{"meta":{"date":"2026-03-04","timezone":"UTC","generated_at":"2026-11-01T00:08:16Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":86,"failed_runs":48,"runs_with_occurrences":48,"runs_with_failure_pattern_matches":45,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":38,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029342028214571008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T23:43:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029337395899404288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T23:25:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8757a11dfd23d564346b23eb62ac01e2353d6d7583f802b18d1c0025b21726e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029337395899404288","occurred_at":"2026-03-04T23:25:13Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T23:55:10.585Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T23:55:12.507Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T23:55:13.792Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=dcd9608e40f4d3412620dde6abf1be2f6e66fb94a15c7a5937888afd3511bbf4\ntime=2026-03-04T23:56:44.857Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9404288/providers/Microsoft.Resources/deployments/dcd9608e40f4d3412620dde6abf1be2f6e66fb94a15c7a5937888afd3511bbf4/operationStatuses/08584289383724665115\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9404288/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9404288/providers/Microsoft.Monitor/accounts/services-j9404288/providers/Microsoft.Authorization/roleAssignments/b072200c-3836-5d7b-a0cc-8762024231bc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T235534Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'd3a4b710-b813-410b-822b-980dd1d8facc', request correlation id 'b27744b3-4c11-44fe-a8ae-a4f561e841f6'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4299/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029335737991696384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4299,"pr_state":"closed","pr_sha":"89abb52bd0ca3634c8e64a287368b6c007666f1e","final_merged_sha":"89abb52bd0ca3634c8e64a287368b6c007666f1e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T23:18:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029335089929785344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T23:16:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4319/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029330890152218624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4319,"pr_state":"closed","pr_sha":"c59e3219a4a59b9af50922acfc3008261b25d09e","final_merged_sha":"244d96cb05aa29b95f158b2263feff5b21b68137","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T22:59:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029324349785247744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T22:33:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"44711e5e97974bda88ef3ba6f09a6cdd5b0ad75eb1f06f05c549f93d3d073de8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029324349785247744","occurred_at":"2026-03-04T22:33:22Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T23:06:15.180Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T23:06:17.640Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T23:06:18.952Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=64274811a5ef1de0087200767591e57def13f2d84e6732be5258beb03413ef2f\ntime=2026-03-04T23:08:20.349Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5247744/providers/Microsoft.Resources/deployments/64274811a5ef1de0087200767591e57def13f2d84e6732be5258beb03413ef2f/operationStatuses/08584289413073485846\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5247744/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5247744/providers/Microsoft.Monitor/accounts/hcps-j5247744/providers/Microsoft.Authorization/roleAssignments/569de546-f40c-5a2d-9e32-a0e0f6144e5c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T230649Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0388b196-e90d-44d2-8660-3f9a774dd473', request correlation id 'fb523f0a-3beb-48dc-9e26-032a91afc0d9'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029323641379885056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T22:30:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d6569ff2d5d65e11ac0b2fbfb9954c8e3c9ddd5e69adf8fc56089c4f1ee3ac09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029323641379885056","occurred_at":"2026-03-04T22:30:33Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T23:01:52.702Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T23:01:55.035Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T23:01:56.362Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=781e055fba988dc0f3fd638ee8d2dc2a1db61385c3a489642509e739bbc05ba3\ntime=2026-03-04T23:03:57.233Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9885056/providers/Microsoft.Resources/deployments/781e055fba988dc0f3fd638ee8d2dc2a1db61385c3a489642509e739bbc05ba3/operationStatuses/08584289415698775263\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9885056/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9885056/providers/Microsoft.Monitor/accounts/services-j9885056/providers/Microsoft.Authorization/roleAssignments/6f3d6dcb-1e92-56b7-93bb-abe94ff265e2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T230248Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '1c621ad6-ca10-49c4-9430-cd0ec461ca86', request correlation id 'b666a097-93df-480c-bffc-f56e803edc29'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029322830893551616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T22:27:20Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4c5206132b24aa1c8caad856beb75cc2b80c4dcc9a5cf7a5f896af12c4632b83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029322830893551616","occurred_at":"2026-03-04T22:27:20Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T22:54:10.500Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T22:54:12.028Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T22:54:13.442Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9c9e8a99c517a55b3fdc1edcd4c7c9781a8fb3bb0cfed133ecd1933c1e92b878\ntime=2026-03-04T22:56:44.608Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j3551616/providers/Microsoft.Resources/deployments/9c9e8a99c517a55b3fdc1edcd4c7c9781a8fb3bb0cfed133ecd1933c1e92b878/operationStatuses/08584289420329641892\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3551616/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3551616/providers/Microsoft.Monitor/accounts/services-j3551616/providers/Microsoft.Authorization/roleAssignments/fd1c4b0e-e117-5355-9d18-185137e542d7\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T225627Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e0b25da4-fc54-4d66-a1fc-0a012dd7c572', request correlation id '64334af4-052b-42c8-a65c-a1a35ff3c555'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3551616/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3551616/providers/Microsoft.Monitor/accounts/hcps-j3551616/providers/Microsoft.Authorization/roleAssignments/63558cc7-12a9-53a5-8ad4-5ad66d246742\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T225628Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '46666170-7963-4ecd-be62-aa6eba00e8bb', request correlation id '64334af4-052b-42c8-a65c-a1a35ff3c555'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029317371709100032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"2f259402b3f6d2f1857a76902fec07cbf1087204","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T22:05:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"21e8701ccdcd9a899f5881917e5993fc155e5b6779276ee4f89ff9a27d19200e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029317371709100032","occurred_at":"2026-03-04T22:05:38Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T22:38:32.982Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T22:38:35.251Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T22:38:36.424Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cb7a03da1653cb23813e75d6c2aaacde0530500ff8410cdafdfababf66e13a58\ntime=2026-03-04T22:41:37.548Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9100032/providers/Microsoft.Resources/deployments/cb7a03da1653cb23813e75d6c2aaacde0530500ff8410cdafdfababf66e13a58/operationStatuses/08584289429697883904\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9100032/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9100032/providers/Microsoft.Monitor/accounts/hcps-j9100032/providers/Microsoft.Authorization/roleAssignments/48d6559e-d792-586f-9fed-e819409a0132\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T224106Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '568ac856-0693-40e6-b9c3-dcbb5ffc3972', request correlation id 'f2f36624-97c0-4e15-9dce-61082732d936'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029311315515805696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4321,"pr_state":"closed","pr_sha":"db95ca2c49d569bc9f7af197ced9db037c7870c7","final_merged_sha":"db95ca2c49d569bc9f7af197ced9db037c7870c7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T21:41:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5df070adc7df4f40572ac33ae7346f8e85323e427c731ef08d1031d7319b3e43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029311315515805696","occurred_at":"2026-03-04T21:41:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T22:07:24.751Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T22:07:27.147Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T22:07:28.320Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4f5ae9d00d4f03e57c42ea2fd4fded5c6ad0d5fd0fccbbaf869b48f9986dcf91\ntime=2026-03-04T22:09:29.365Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5805696/providers/Microsoft.Resources/deployments/4f5ae9d00d4f03e57c42ea2fd4fded5c6ad0d5fd0fccbbaf869b48f9986dcf91/operationStatuses/08584289448378848505\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5805696/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5805696/providers/Microsoft.Monitor/accounts/services-j5805696/providers/Microsoft.Authorization/roleAssignments/8de06612-55e5-5107-b071-362a6c69e1b8\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T220741Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '39bbf176-8363-489e-8417-42cabcb236e5', request correlation id '985c1e1f-f300-420f-ba2f-46fb722f2c02'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029307974203215872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"0381114f65dcac7eed12f92506178367a8e55c4d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T21:28:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d74f1103aa2fa9f3c18f35113d8f00e9bd008982303a47bbc4322d05197d02cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029307974203215872","occurred_at":"2026-03-04T21:28:18Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T21:55:00.564Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T21:55:02.462Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T21:55:04.523Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=16534738d0ceec5d3ed4b7a0cc105ed7a31f056e999f0a7a110a876e07c59dd7\ntime=2026-03-04T21:56:35.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j3215872/providers/Microsoft.Resources/deployments/16534738d0ceec5d3ed4b7a0cc105ed7a31f056e999f0a7a110a876e07c59dd7/operationStatuses/08584289455824981586\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3215872/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3215872/providers/Microsoft.Monitor/accounts/hcps-j3215872/providers/Microsoft.Authorization/roleAssignments/c3eea95f-2a47-598c-8c99-a4f0f6dad3e0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T215525Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7d6b2bd0-20db-4f59-b3ae-227ca9b2df8e', request correlation id 'a4acfe60-8934-4643-b4d0-bc7bb3dfe921'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029306092927848448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4279,"pr_state":"closed","pr_sha":"9bce19c0b253a1664b88a85024fa247529209aff","final_merged_sha":"efa8e746c645a87051488818bbf21f67fb2329fd","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T21:20:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029299212662870016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T20:53:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029295810885193728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4279,"pr_state":"closed","pr_sha":"9bce19c0b253a1664b88a85024fa247529209aff","final_merged_sha":"efa8e746c645a87051488818bbf21f67fb2329fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T20:39:58Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6acb8f34666d020a223d705b34f9cbe4b3acf97812b613df62dfab98f38ae438","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029295810885193728","occurred_at":"2026-03-04T20:39:58Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T21:07:20.042Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T21:07:21.971Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T21:07:23.065Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e7aebd1a20481f960a46afad379b5c7398aaf21867cdf67fb3e0b428c9ad66ad\ntime=2026-03-04T21:08:53.586Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5193728/providers/Microsoft.Resources/deployments/e7aebd1a20481f960a46afad379b5c7398aaf21867cdf67fb3e0b428c9ad66ad/operationStatuses/08584289484430528698\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5193728/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5193728/providers/Microsoft.Monitor/accounts/hcps-j5193728/providers/Microsoft.Authorization/roleAssignments/2a533807-f55e-57bb-a22c-ea01a80ef8c5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T210743Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '53f9b98c-3b27-46ce-9a62-576d11993cf9', request correlation id '12e72819-3dfc-4fb6-9140-cb44dab82bb5'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5193728/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5193728/providers/Microsoft.Monitor/accounts/services-j5193728/providers/Microsoft.Authorization/roleAssignments/5dab1394-c7a3-5866-9dc2-e6f5ed27dc2c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T210753Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '15bcb1c0-5980-48aa-b69c-d227395e9a0b', request correlation id '12e72819-3dfc-4fb6-9140-cb44dab82bb5'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4320/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029289035045801984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4320,"pr_state":"closed","pr_sha":"013c94061eca5377e4335bb6b728b77300bfc97a","final_merged_sha":"0bc9efd2e7e3d329a191fc2722583dc5b88a9b14","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T20:13:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029285442439876608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T19:58:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029284708868689920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T19:55:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c1ecfa74b6116259e4a39f73ff59529653f2395c2482e0ba83837d9929169d70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029284708868689920","occurred_at":"2026-03-04T19:55:51Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T20:28:33.469Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T20:28:35.632Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T20:28:36.743Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1e2746236a223cc82826d40c7a221258c815eaa771d177b618b6eedc14af321e\ntime=2026-03-04T20:30:37.678Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8689920/providers/Microsoft.Resources/deployments/1e2746236a223cc82826d40c7a221258c815eaa771d177b618b6eedc14af321e/operationStatuses/08584289507694457557\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8689920/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8689920/providers/Microsoft.Monitor/accounts/services-j8689920/providers/Microsoft.Authorization/roleAssignments/6d179d95-0fb6-5120-81e3-0bbfe064445a\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T202852Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '917ac4ea-b671-4207-bb9e-ff38aab9941d', request correlation id '3fdbb3ee-b775-4232-8b7d-49154b8b744f'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029283978019606528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T19:52:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029283890769694720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4289,"pr_state":"closed","pr_sha":"f879b10dc600aa4bc317f4550e395d2e07047a53","final_merged_sha":"f879b10dc600aa4bc317f4550e395d2e07047a53","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T19:52:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029282409387659264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"b60ce1f30ada0d35e621e79382aa3e59e0ec4f23","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T19:46:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"014d3de5be3b018832ff6aea6c621ffc013717e93a35b74f94ced3983dd4bd64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029282409387659264","occurred_at":"2026-03-04T19:46:43Z","failed_at":"e2e","signature_id":"144930c58f7349ab1a03e12c7cf2cf56b0d115c6a050ad67cd40a3ad73b09439","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:232]: expected 2 nodes to still have original taint key1=value1:NoSchedule\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:232]: expected 2 nodes to still have original taint key1=value1:NoSchedule\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4052a36339c4f620637bcbd6117118ce38308a1496ff3a03cd71b765fb5b210a","failure_pattern":"expected 2 nodes to still have original taint key1=value1:NoSchedule","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:232]: expected 2 nodes to still have original taint key1=value1:NoSchedule"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4052a36339c4f620637bcbd6117118ce38308a1496ff3a03cd71b765fb5b210a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4285/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279814338220032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4285,"pr_state":"closed","pr_sha":"b61beb928788a2673941120ce3c13c31afe9b930","final_merged_sha":"b61beb928788a2673941120ce3c13c31afe9b930","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T19:36:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4198,"pr_state":"open","pr_sha":"afb6057d0dfa7edde74fcd138854adea2ce39897","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T19:33:47Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"2dd4273395c5883d47fb89cf9a934face8a7a46cf97e690d3951c0c0668cb0e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"ed27907a3200fb4cdab0338336ce58d5006f1630e11f7c30d3791dc00cf2d4ed","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a0f800\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-ds7hg6, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ds7hg6\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-ds7hg6, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ds7hg6\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000784f80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-ds7hg6\",\n },\n \u003c*fmt.wrapError | 0xc00017b2c0\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ds7hg6\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a0f800\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-ds7hg6, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ds7hg6\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-ds7hg6, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ds7hg6\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000784f80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-ds7hg6\",\n },\n \u003c*fmt.wrapError | 0xc00017b2c0\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-ds7hg6\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded"}},{"row_id":"35ba21b848ea921396e47665432037c4a2c8287bf7f2a834dceff229c6710874","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"54c38e2b642fc5eda492fcce9c17e1b599c5fd2f3bcdf54f0265ccc23a1b95d6","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b92c00\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-hx4qt7, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hx4qt7\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-hx4qt7, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-hx4qt7\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000653b60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-hx4qt7\",\n },\n \u003c*fmt.wrapError | 0xc0003c8fc0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-hx4qt7\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b92c00\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-hx4qt7, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hx4qt7\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-hx4qt7, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-hx4qt7\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000653b60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-hx4qt7\",\n },\n \u003c*fmt.wrapError | 0xc0003c8fc0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-hx4qt7\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded"}},{"row_id":"3974bd8ac931a716e699626002d74561f7eaf63892e0302785bd39c16afbde0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"8c7c9cc4d0ab049c78c2480694263d677d4135d5db1600e2cf4ab7265eccea0d","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000efb0b0\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-n9h7f8\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012c5340\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8\",\n },\n \u003c*fmt.wrapError | 0xc00080ab00\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-n9h7f8\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000efb0b0\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-n9h7f8\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012c5340\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8\",\n },\n \u003c*fmt.wrapError | 0xc00080ab00\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-n9h7f8\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded"}},{"row_id":"3c3dae85638fc8f742c80b4d1230f5188c58071bdf72c585fce5d2da64243d49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"ca45b03900905d1252c4f86ee95cb3288b7b44efa4aa105b95a3252af569d236","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b8caa0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-ws5kq6\": PUT https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ws5kq6\\\": PUT https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b8ca80\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ws5kq6\\\": PUT https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010f5650\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"345\"],\n \"X-Envoy-Upstream-Service-Time\": [\"76\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"9eda12b4-bb47-465d-8806-c9ac50df5fa4\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 21:03:05 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b8c9a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b8caa0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-ws5kq6\": PUT https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ws5kq6\\\": PUT https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b8ca80\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-ws5kq6\\\": PUT https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws5kq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010f5650\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"345\"],\n \"X-Envoy-Upstream-Service-Time\": [\"76\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"9eda12b4-bb47-465d-8806-c9ac50df5fa4\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 21:03:05 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b8c9a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"4d7644681bf0068e6c0414c26c5e88a9d883b50ec5f9d16979fc07333c620aff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"4a5da64bf38cf75144c4d11f15cf0d792a57b71ae88d170859c4ea76a5388a81","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00146c060\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-4xdqrt, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-4xdqrt\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-4xdqrt, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4xdqrt\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005b23c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-4xdqrt\",\n },\n \u003c*fmt.wrapError | 0xc000cdc020\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4xdqrt\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00146c060\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-4xdqrt, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-4xdqrt\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-4xdqrt, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4xdqrt\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005b23c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-4xdqrt\",\n },\n \u003c*fmt.wrapError | 0xc000cdc020\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-4xdqrt\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded"}},{"row_id":"55d11c44099f9eac73ca539e650c9171b2f8b384458c0d12df4f0a71c2c73df2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"03652fca43e45084cf73c1b3b4bf8c0cae3e0b04ee3ffa9a3728d5975f4514a0","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000850408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list96w5f7\" in resourcegroup=\"ea-list-hhrk4k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009ee640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list96w5f7\\\" in resourcegroup=\\\"ea-list-hhrk4k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009ee620\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list96w5f7\\\" in resourcegroup=\\\"ea-list-hhrk4k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009ee600\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-list96w5f7\\\" in resourcegroup=\\\"ea-list-hhrk4k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0260b0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000850408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list96w5f7\" in resourcegroup=\"ea-list-hhrk4k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009ee640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list96w5f7\\\" in resourcegroup=\\\"ea-list-hhrk4k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009ee620\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list96w5f7\\\" in resourcegroup=\\\"ea-list-hhrk4k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009ee600\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-list96w5f7\\\" in resourcegroup=\\\"ea-list-hhrk4k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0260b0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list96w5f7\" in resourcegroup=\"ea-list-hhrk4k\" to finish deleting: context canceled"}},{"row_id":"5ad0e1b2febdcbefca8d123b57ff1d8f1ea5ade141e121612215a68b63d3a2ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"f6a84dfceee381637a8eb7447d8c3881a0534d51a893eb9b7402e1629075f38c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e66030\u003e: \n failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-blmsns\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-blmsns\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014c5270\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e66030\u003e: \n failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-blmsns\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-blmsns\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014c5270\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-blmsns\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-blmsns, error: context deadline exceeded"}},{"row_id":"86b2929ba1151750044e4269e9ef81af88007ef5a9ffe3dea019fbc5d5b719c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"837df6cb2f051150846efd2f45bf384f6f5e8e221d631364701cfb40e37d2ee0","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:92]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a08d00\u003e: \n failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011615c0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"39a12661-31ec-4bb4-9474-f1a1fecdcb20\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 21:03:38 GMT\",\n ],\n \"Content-Length\": [\"336\"],\n \"X-Envoy-Upstream-Service-Time\": [\"24\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a08c20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 110, 111, 100, 101, 112, 111, 111, 108, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 102, 119, 113, 56, 107, 102, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 110, 112, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 104, 99, 112, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 336,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j7517312.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/974ebd46-8ad3-41e3-afef-7...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:92]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a08d00\u003e: \n failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-fwq8kf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011615c0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"39a12661-31ec-4bb4-9474-f1a1fecdcb20\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 21:03:38 GMT\",\n ],\n \"Content-Length\": [\"336\"],\n \"X-Envoy-Upstream-Service-Time\": [\"24\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a08c20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 110, 111, 100, 101, 112, 111, 111, 108, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 102, 119, 113, 56, 107, 102, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 110, 112, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 104, 99, 112, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 336,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j7517312.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/974ebd46-8ad3-41e3-afef-7...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"9094bd90f781b3e70d7f7d522b6b329397e8d5b878087cec14d8d2e0885dec84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"23c95e94b86476d966345e3169c6a2673861b8d221b8eb496f143df064a4d8b6","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001482e40\u003e: \n failed to create NodePool autoscale-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-np in resource group np-autoscaling-52w4cj, error: failed waiting for nodepool=\"autoscale-np\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-52w4cj\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool autoscale-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-np in resource group np-autoscaling-52w4cj, error: failed waiting for nodepool=\\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52w4cj\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001101540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-np in resource group np-autoscaling-52w4cj\",\n },\n \u003c*fmt.wrapError | 0xc00150a580\u003e{\n msg: \"failed waiting for nodepool=\\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52w4cj\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001482e40\u003e: \n failed to create NodePool autoscale-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-np in resource group np-autoscaling-52w4cj, error: failed waiting for nodepool=\"autoscale-np\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-52w4cj\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool autoscale-np, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-np in resource group np-autoscaling-52w4cj, error: failed waiting for nodepool=\\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52w4cj\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001101540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-np in resource group np-autoscaling-52w4cj\",\n },\n \u003c*fmt.wrapError | 0xc00150a580\u003e{\n msg: \"failed waiting for nodepool=\\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-52w4cj\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded"}},{"row_id":"af57b9b93a321443a0731f6fa688721affbf02b75145897cdca9bb101ffdfaf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"281d78683098cc1429a6e8a6ce78d4a540d170e3d38088b005cd5386c1636ef2","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:156]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009807c0\u003e: \n failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"accepted\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015252f0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"dc4fe3fc-5713-4a7a-848c-01bfb109e432\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 21:19:21 GMT\",\n ],\n \"Content-Length\": [\"323\"],\n \"X-Envoy-Upstream-Service-Time\": [\"27\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009806c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 97, 99, 99, 101, 112, 116, 101, 100, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 114, 103, 45, 99, 108, 117, 115, 116, 101, 114, 45, 98, 97, 99, 107, 45, 118, 101, 114, 115, 105, 111, 110, 45, 55, 98, 112, 55, 114, 57, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 99, 108, 117, 115, 116, 101, 114, 45, 118, 101, 114, 45, 52, 45, 49, 57, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 323,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j7517312.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcp...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:156]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009807c0\u003e: \n failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"accepted\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j7517312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015252f0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"dc4fe3fc-5713-4a7a-848c-01bfb109e432\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 21:19:21 GMT\",\n ],\n \"Content-Length\": [\"323\"],\n \"X-Envoy-Upstream-Service-Time\": [\"27\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009806c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 97, 99, 99, 101, 112, 116, 101, 100, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 114, 103, 45, 99, 108, 117, 115, 116, 101, 114, 45, 98, 97, 99, 107, 45, 118, 101, 114, 115, 105, 111, 110, 45, 55, 98, 112, 55, 114, 57, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 99, 108, 117, 115, 116, 101, 114, 45, 118, 101, 114, 45, 52, 45, 49, 57, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 323,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j7517312.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-7bp7r9/providers/Microsoft.RedHatOpenShift/hcp...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ef61d2ea77a5b1bf60c86a80af6cbbe3e5d1d9a5dbedee1bba1811c574bcb119","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"bc1a6e1770bd22042676306b6e2f7eda68ee40ca63cd9d5e120b0c6db2c0ff1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029279155077517312","occurred_at":"2026-03-04T19:33:47Z","failed_at":"e2e","signature_id":"84e2575782101473312d2fbaf58e641c20f7c610a26bc81742cac3f22e258637","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001678de0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-7779tn, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7779tn\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-7779tn, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7779tn\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00176feb0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-7779tn\",\n },\n \u003c*fmt.wrapError | 0xc0004ec8c0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7779tn\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001678de0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-7779tn, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7779tn\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-7779tn, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7779tn\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00176feb0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-7779tn\",\n },\n \u003c*fmt.wrapError | 0xc0004ec8c0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7779tn\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-n9h7f8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n9h7f8\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","ef61d2ea77a5b1bf60c86a80af6cbbe3e5d1d9a5dbedee1bba1811c574bcb119"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029278147718942720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4284,"pr_state":"closed","pr_sha":"efd28c631288d18973d727e83e9a2b147a02894e","final_merged_sha":"efd28c631288d18973d727e83e9a2b147a02894e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T19:29:47Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"696e3641c72261599c9eeaf2b6489486c241f3106aacbefd1b6c9f4d714afb52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029278147718942720","occurred_at":"2026-03-04T19:29:47Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T20:03:27.202Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T20:03:29.325Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T20:03:30.674Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=114589f546f36207463d98126d188d1970f8c06b6e43e7fa9056fbaea567b080\ntime=2026-03-04T20:05:31.658Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8942720/providers/Microsoft.Resources/deployments/114589f546f36207463d98126d188d1970f8c06b6e43e7fa9056fbaea567b080/operationStatuses/08584289522756264799\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8942720/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8942720/providers/Microsoft.Monitor/accounts/services-j8942720/providers/Microsoft.Authorization/roleAssignments/fc4772bd-3e5c-5b5d-a99a-a56d1dcc3d3f\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T200344Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'fd1bb717-3e55-473a-9f0c-063e115fc731', request correlation id '50eabdd7-ca4b-4468-a1ad-353af33ce524'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8942720/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8942720/providers/Microsoft.Monitor/accounts/hcps-j8942720/providers/Microsoft.Authorization/roleAssignments/ec8367bb-50a2-5149-b952-50f32d1ef6a8\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T200352Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'ed6d8a05-f41f-4c70-866a-a57600402054', request correlation id '50eabdd7-ca4b-4468-a1ad-353af33ce524'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4319/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029277847662628864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4319,"pr_state":"closed","pr_sha":"c59e3219a4a59b9af50922acfc3008261b25d09e","final_merged_sha":"244d96cb05aa29b95f158b2263feff5b21b68137","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T19:28:24Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b5ef8741213051d6d6fbc65eff10219e8d580533ade73f7316bf7c53da0c73e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4319/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029277847662628864","occurred_at":"2026-03-04T19:28:24Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T19:58:35.089Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T19:58:37.792Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T19:58:39.149Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6a16024d1e9f409c92033f499d2defc00d92c0dac6a1acd92f07ad03ec094979\ntime=2026-03-04T20:00:40.282Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2628864/providers/Microsoft.Resources/deployments/6a16024d1e9f409c92033f499d2defc00d92c0dac6a1acd92f07ad03ec094979/operationStatuses/08584289525671888074\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2628864/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2628864/providers/Microsoft.Monitor/accounts/services-j2628864/providers/Microsoft.Authorization/roleAssignments/8b284b8a-20cf-5023-b694-cb3fbe719ae6\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T195854Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0fe8dd9e-695e-43e9-8a26-426c7e5d97e3', request correlation id 'df8e2676-c454-4096-9185-ec65a8b0ab6e'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029267675275857920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T18:48:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"807d0f99842e441c3b467836575aed2619fb15e393c3ad055bf1d676a66bf8ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029267675275857920","occurred_at":"2026-03-04T18:48:10Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T19:18:07.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T19:18:09.297Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T19:18:10.883Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1d977f7d908bb0d6fe0e7d76ab760a2d4bb50f8189660bd1895b23b0772b8a4c\ntime=2026-03-04T19:20:11.968Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5857920/providers/Microsoft.Resources/deployments/1d977f7d908bb0d6fe0e7d76ab760a2d4bb50f8189660bd1895b23b0772b8a4c/operationStatuses/08584289549956731661\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5857920/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5857920/providers/Microsoft.Monitor/accounts/hcps-j5857920/providers/Microsoft.Authorization/roleAssignments/18509be4-9027-57b5-8639-cbaf540b3286\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T191831Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '90c10169-f367-485e-94d9-aa30eb3ef32b', request correlation id '929bfc43-4321-4b7d-8d07-5f84a153792b'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029264688117714944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4303,"pr_state":"closed","pr_sha":"7148a34a98c3daf4639ce203a1af6126324406ca","final_merged_sha":"7148a34a98c3daf4639ce203a1af6126324406ca","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T18:36:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029253148907933696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4310,"pr_state":"closed","pr_sha":"4b5ae236d9d72c2b4c53c9d7ce5e9b67247cf9bc","final_merged_sha":"4b5ae236d9d72c2b4c53c9d7ce5e9b67247cf9bc","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T17:50:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"309c5a6f37ec595f8b6028601dc08666d1704fcb89bad06e4f810e3523788d97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029253148907933696","occurred_at":"2026-03-04T17:50:27Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T18:35:02.105Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T18:35:04.165Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T18:35:05.424Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ad1d26434c191aa2798b467a2b633dc3445d6b35562229560a1c41b231bc6af6\ntime=2026-03-04T18:37:06.921Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7933696/providers/Microsoft.Resources/deployments/ad1d26434c191aa2798b467a2b633dc3445d6b35562229560a1c41b231bc6af6/operationStatuses/08584289575808313886\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7933696/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7933696/providers/Microsoft.Monitor/accounts/services-j7933696/providers/Microsoft.Authorization/roleAssignments/5ee749ef-c3ec-5b35-b998-da7137950502\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T183615Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c3b68396-ddec-46cd-b39a-9b13ceb9a629', request correlation id '4f15cad2-9417-4640-a9e3-8ee022b1de7e'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4315/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029245465958027264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4315,"pr_state":"closed","pr_sha":"0b3a4559c5f1ad0b6eb59ba665c2228803c96004","final_merged_sha":"0b3a4559c5f1ad0b6eb59ba665c2228803c96004","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T17:19:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029244728024764416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4314,"pr_state":"closed","pr_sha":"35511ce9c20977aff9135f8686faa6039565f5ce","final_merged_sha":"35511ce9c20977aff9135f8686faa6039565f5ce","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T17:16:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f5c9b533fb6bdd1f2b185a913eff9d683b5f22d44d4931e0e87a4dedcf65872e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029244728024764416","occurred_at":"2026-03-04T17:16:59Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T17:49:05.546Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T17:49:08.081Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T17:49:09.262Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8e060c96442b359ddd0d3602923fedefc46092bc3a046ecc9e71e02f11d53b75\ntime=2026-03-04T17:51:10.126Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4764416/providers/Microsoft.Resources/deployments/8e060c96442b359ddd0d3602923fedefc46092bc3a046ecc9e71e02f11d53b75/operationStatuses/08584289603369475030\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4764416/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4764416/providers/Microsoft.Monitor/accounts/services-j4764416/providers/Microsoft.Authorization/roleAssignments/51e70bce-e48e-5148-b2c5-6ac918d25157\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T174924Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '82ff6217-fc88-488c-a5bc-e72aa66bab19', request correlation id '1676f6e7-e03a-4094-94ea-ef92ca94cc46'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4764416/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4764416/providers/Microsoft.Monitor/accounts/hcps-j4764416/providers/Microsoft.Authorization/roleAssignments/939d22ac-0fcd-581c-a376-a04f2a0017ff\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T174921Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5f75a6f5-273a-469e-af24-e6d151d5f962', request correlation id '1676f6e7-e03a-4094-94ea-ef92ca94cc46'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029235566347816960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4181,"pr_state":"closed","pr_sha":"52c9c6919275db96790f615d00f5534ffc2ee338","final_merged_sha":"0ac5e198ed7921a253ae14ad2c7ea366e71f81ce","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T16:40:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"36df3100cabfc5e3b7b21f2e89ffe50d25c061e41d6202e0a454f5d47feba685","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029235566347816960","occurred_at":"2026-03-04T16:40:34Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T17:12:01.264Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T17:12:03.132Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T17:12:04.363Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4367a7d3e2f3e375f64a27aa555715150c21e58b0224619e5ab6f81e59b76d47\ntime=2026-03-04T17:13:35.222Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7816960/providers/Microsoft.Resources/deployments/4367a7d3e2f3e375f64a27aa555715150c21e58b0224619e5ab6f81e59b76d47/operationStatuses/08584289625617683621\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7816960/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7816960/providers/Microsoft.Monitor/accounts/hcps-j7816960/providers/Microsoft.Authorization/roleAssignments/2d6818b1-8d22-58e9-8a48-c9a74b129989\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T171219Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9a1c3e98-ed1f-4270-b1ea-1f180744dbf9', request correlation id 'beeaf399-2261-4d54-ab5f-3e86f1af80bb'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7816960/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7816960/providers/Microsoft.Monitor/accounts/services-j7816960/providers/Microsoft.Authorization/roleAssignments/8b526356-7b87-5bf7-9086-1924759a5eee\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T171228Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '24e5a517-95bb-4729-8007-d3e1e7a5b105', request correlation id 'beeaf399-2261-4d54-ab5f-3e86f1af80bb'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029234058780741632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"b60ce1f30ada0d35e621e79382aa3e59e0ec4f23","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T16:34:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a8b3e9cabc4964a350a2c0cb99f1fe824a5af6285adc3b5f59de4abe5dd7988c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029234058780741632","occurred_at":"2026-03-04T16:34:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T17:15:18.278Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T17:15:20.533Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T17:15:21.781Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d715d7f2f86c8d9c8a18d69fd57d4c08c3094631b0ab9e92abd7ce843db5585d\ntime=2026-03-04T17:16:52.478Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0741632/providers/Microsoft.Resources/deployments/d715d7f2f86c8d9c8a18d69fd57d4c08c3094631b0ab9e92abd7ce843db5585d/operationStatuses/08584289623644387667\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0741632/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0741632/providers/Microsoft.Monitor/accounts/services-j0741632/providers/Microsoft.Authorization/roleAssignments/e0314240-293c-5d24-b673-615952444f06\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T171537Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'bceb4411-ee20-4276-9faa-436d38bb2a38', request correlation id '8fdb6523-a3ec-47db-9e20-290098faf894'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0741632/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0741632/providers/Microsoft.Monitor/accounts/hcps-j0741632/providers/Microsoft.Authorization/roleAssignments/722ecb59-4670-531e-8c1d-3d1f218cad3f\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T171613Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '896a8848-22a3-4ec8-9ac9-1a25af81a7da', request correlation id '8fdb6523-a3ec-47db-9e20-290098faf894'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029234007845113856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4318,"pr_state":"closed","pr_sha":"6e48d061192c64d4f4c380a842e7b5388e92ae73","final_merged_sha":"6e48d061192c64d4f4c380a842e7b5388e92ae73","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T16:34:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029232786476371968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"a59797e2511a6fdf340ebd55dfb548aa7b6cb947","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T16:29:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d8fa4e969d7961541d4ae13a2185bbb4aef0cc6bff6ae5923bac8d5c4b98b243","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029232786476371968","occurred_at":"2026-03-04T16:29:32Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T17:03:57.931Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T17:04:00.330Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T17:04:01.659Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9c599b7af13588fd58ca5a46b5550cc3c3631f025471032cd316fd5e7b184c20\ntime=2026-03-04T17:05:32.407Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6371968/providers/Microsoft.Resources/deployments/9c599b7af13588fd58ca5a46b5550cc3c3631f025471032cd316fd5e7b184c20/operationStatuses/08584289630446024430\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6371968/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6371968/providers/Microsoft.Monitor/accounts/services-j6371968/providers/Microsoft.Authorization/roleAssignments/64c6e2a3-0acf-5ef1-b40a-cfe8e721ec65\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T170414Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '3ccc6d38-1264-4093-9f38-b4e19ded1fd4', request correlation id '51d6ed71-fac7-4caa-9d47-d4eb5b2c8863'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6371968/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6371968/providers/Microsoft.Monitor/accounts/hcps-j6371968/providers/Microsoft.Authorization/roleAssignments/18358df3-69f2-5b96-8489-24a2425bf3c5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T170416Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'adb321d9-6ff6-46a1-9550-baabb063e081', request correlation id '51d6ed71-fac7-4caa-9d47-d4eb5b2c8863'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029230167754280960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"2bbe0eb34fece99f725ae46b78fbfce1ad7ef605","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T16:19:07Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"df4d896f3d80e4d1a16e7af56bff826c65957da63b8c4f9aa69f7766748a44cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029230167754280960","occurred_at":"2026-03-04T16:19:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029227711465000960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T16:09:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4317/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029221812633604096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4317,"pr_state":"closed","pr_sha":"bc6c3748d9fdd3aada9468883a504bfeac11c522","final_merged_sha":"3832c3f226ce4016ff6187e06101b490c14f1565","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T15:45:21Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0b1aae687286171703e6e0f59e9b305b6019eabd9696543f1c0e691f3c94dded","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4317/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029221812633604096","occurred_at":"2026-03-04T15:45:21Z","failed_at":"provision","signature_id":"2c0cdfff6d04e9c5eda9a909dd3880853c31f0f9acd300f97287b842d126476c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-04T16:42:54.462Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-04T16:42:55.072Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:52268a48-3ba3-4eb7-bae0-330881d68626 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j3604096.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j3604096] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:062f0656-796b-42b2-99c8-7ecf6461937b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T16:42:55.072Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:42:55.379Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-03-04T16:42:55.788Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-03-04T16:42:55.788Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:42:56.022Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:48:03.838Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource not ready, name: maestro, kind: Deployment, status: InProgress\\nresource not ready, name: maestro-db, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T16:48:03.838Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:48:03.838Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:48:08.035Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy release=maestro-server namespace=maestro status=failed description=\"Release \\\"maestro-server\\\" failed: resource not ready, name: maestro, kind: Deployment, status: InProgress\\nresource not ready, name: maestro-db, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[clientId:52268a48-3ba3-4eb7-bae0-330881d68626 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j3604096.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j3604096] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:062f0656-796b-42b2-99c8-7ecf6461937b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T16:48:08.037Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yUQW/TQBCF7/4VI58SKa5CaKsqqIdSOCFQJBCHIg7r3Ye9yXrXmh0nCsqPR04oMa3TGG7r2e+NZ9+MJnEQYsTQsEakWzJKlKjcYRTyJbR8sN7Mo7D1xYQOoU+qwp8Q1vD7SKyVfgyPvyVEROln8Npq3GkdGi/phNJKIQqHv44HdgGONgq8fA2uqXDvlK06XGbyHtXvP3S5gms9jCyhnJS6hF4NEgzLWkHY6jgM7n3UO9QubCucs6yXG5KyDlEKRsxyhloVTsW+cu80Lxrn3lpvrC/2usa5LD9+PxU0Uga2P5XY4BfBWb1tKeVc2GS50it4868y7Zoo4Cz2dGOA/NEUOeHLy+KTnXxR5oOU/2EQo7BReH+XLUPe1nx28D4GbyVwt/+tU+BeiWacGalnTL9tC4Dbx8CL1fuKW8jgh2qcnEy74LC2Bnz/bOCOx+T7m2TV5GAPQXzfbpeY7GhTgkFiK0RRVU05ZAN4GhklaMOjdDadXWfT19n08sur6/nlbH519ZCO6eKCTjI38+nNQzoeJztaButpZb25td6DOzsxeDquwu4OfLr8kh3VHNrbY6ETqoM5wMOSTMiFYkIVYlRFmzKwAVO+7TzeIOpfAQAA//+ztJdiuQUAAA==\"\ntime=2026-03-04T16:48:08.040Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:48:08.041Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: maestro, kind: Deployment, status: InProgress\\nresource not ready, name: maestro-db, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T16:50:08.042Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-04T16:50:09.026Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:52268a48-3ba3-4eb7-bae0-330881d68626 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j3604096.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j3604096] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:062f0656-796b-42b2-99c8-7ecf6461937b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T16:50:09.026Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:50:09.534Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-03-04T16:50:09.802Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:50:09.802Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:50:10.016Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy numVersions=1\ntime=2026-03-04T16:50:11.026Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:50:11.026Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=maestro name=maestro\ntime=2026-03-04T16:50:11.399Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=PersistentVolumeClaim\" namespace=maestro name=maestro-db\ntime=2026-03-04T16:50:11.762Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-grpc\ntime=2026-03-04T16:50:12.092Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-healthcheck\ntime=2026-03-04T16:50:12.212Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro\ntime=2026-03-04T16:50:12.330Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-metrics\ntime=2026-03-04T16:50:12.451Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-db\ntime=2026-03-04T16:50:12.572Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=maestro name=maestro\ntime=2026-03-04T16:50:12.918Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=maestro name=maestro-db\ntime=2026-03-04T16:50:13.038Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=maestro name=postgres-breakglass\ntime=2026-03-04T16:50:13.158Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=maestro name=pull-binding\ntime=2026-03-04T16:50:13.231Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=72.209865ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/acrpull.microsoft.com/v1beta2/namespaces/maestro/acrpullbindings/pull-binding\ntime=2026-03-04T16:50:13.431Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=139.293289ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/acrpull.microsoft.com/v1beta2/namespaces/maestro/acrpullbindings/pull-binding\ntime=2026-03-04T16:50:13.697Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-backend\ntime=2026-03-04T16:50:13.831Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=74.208758ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-backend\ntime=2026-03-04T16:50:14.148Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-cluster-service\ntime=2026-03-04T16:50:14.349Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-maestro-to-db\ntime=2026-03-04T16:50:14.431Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=81.237342ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-maestro-to-db\ntime=2026-03-04T16:50:14.631Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=140.336941ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-maestro-to-db\ntime=2026-03-04T16:50:14.761Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-metrics\ntime=2026-03-04T16:50:14.831Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=70.238372ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-metrics\ntime=2026-03-04T16:50:15.031Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=141.282922ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-metrics\ntime=2026-03-04T16:50:15.106Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-nothing\ntime=2026-03-04T16:50:15.231Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=125.220112ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-nothing\ntime=2026-03-04T16:50:15.431Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=141.331893ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-nothing\ntime=2026-03-04T16:50:15.496Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-registration-job-to-maestro\ntime=2026-03-04T16:50:15.630Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=134.230097ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-registration-job-to-maestro\ntime=2026-03-04T16:50:15.831Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=140.27005ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-registration-job-to-maestro\ntime=2026-03-04T16:50:15.899Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=maestro name=default\ntime=2026-03-04T16:50:16.030Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=131.260133ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1beta1/namespaces/maestro/peerauthentications/default\ntime=2026-03-04T16:50:16.230Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=137.247314ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1beta1/namespaces/maestro/peerauthentications/default\ntime=2026-03-04T16:50:16.489Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=maestro name=maestro\ntime=2026-03-04T16:50:16.631Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=82.198643ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/secrets-store.csi.x-k8s.io/v1/namespaces/maestro/secretproviderclasses/maestro\ntime=2026-03-04T16:50:16.901Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=maestro name=maestro-server\ntime=2026-03-04T16:50:17.030Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=66.138743ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/monitoring.coreos.com/v1/namespaces/maestro/servicemonitors/maestro-server\ntime=2026-03-04T16:50:17.315Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-03-04T16:50:17.315Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:50:17.482Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy numVersions=1\ntime=2026-03-04T16:55:21.909Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T16:55:21.909Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:55:21.909Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:55:23.755Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy release=maestro-server namespace=maestro status=failed description=\"Upgrade \\\"maestro-server\\\" failed: resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[clientId:52268a48-3ba3-4eb7-bae0-330881d68626 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j3604096.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j3604096] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:062f0656-796b-42b2-99c8-7ecf6461937b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T16:55:23.757Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5SUQWvbMBTH7/4UwqcE7JKmawcZPXTdTmMjsLFDxw6y9J+tRNYzT88JGfnww0nbZK1DnaOffr8/T7L0Eg9RjEgtG0R1q6wWLbrwGFGxgJEvLthZFHahzNS+9E3XeC5hhbCrxEabp/L4V6KUUuldKxWx+6vFUZiTd2aTZirV3tM6L7RZItiuUGtEYUqzAZrxbRRwHsErZ3Cu/sjmQrktzpYh7Ew8VwsklQvluRqjdFF4t5YvqOh6fvZfRc0B7uIQxJmd00EWf3TrpYf/vj+/rxScEB8Bu5MFn1bujKE2/J/ZwxrGuUz/L5mDo4uCID/JtzXuvXb1m9YnNJ42Nd5qtJcbEtlQlJIR84Khl6XXse9i7Pc4Z1o5C75/RZ065ONmSm7MMLKC9lKZCmY5SBiWevra98D9r8rwvPX+owv28SE0rfd5cfh+EpLfH5JlW4ADBPFzN11islXrCgwlrkYUXTeqgKyBoEZWC7ryKJ1Opjf55CqfvPtxeTO7nswu3z+kY3VxoU4y17Pp1UM6HidbtSAX1NIFe+tCAB/NRArqMAqPZ+DL4ZdsVcPUrR4azVRDdg8PC8mUpzJTNWLUZRdJbMGq2Bxt3iKafwEAAP//n0DhhrkFAAA=\"\ntime=2026-03-04T16:55:23.761Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:55:23.761Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T16:57:23.762Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-04T16:57:24.576Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:52268a48-3ba3-4eb7-bae0-330881d68626 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j3604096.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j3604096] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:062f0656-796b-42b2-99c8-7ecf6461937b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T16:57:24.576Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:57:25.037Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-03-04T16:57:25.540Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:57:25.540Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:57:25.801Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy numVersions=2\ntime=2026-03-04T16:57:26.867Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:57:26.867Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=maestro name=maestro\ntime=2026-03-04T16:57:27.471Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=PersistentVolumeClaim\" namespace=maestro name=maestro-db\ntime=2026-03-04T16:57:27.984Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-grpc\ntime=2026-03-04T16:57:28.512Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-healthcheck\ntime=2026-03-04T16:57:28.632Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro\ntime=2026-03-04T16:57:28.756Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-metrics\ntime=2026-03-04T16:57:28.876Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=maestro name=maestro-db\ntime=2026-03-04T16:57:28.997Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=maestro name=maestro\ntime=2026-03-04T16:57:29.528Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=maestro name=maestro-db\ntime=2026-03-04T16:57:29.659Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=maestro name=postgres-breakglass\ntime=2026-03-04T16:57:29.783Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=maestro name=pull-binding\ntime=2026-03-04T16:57:30.288Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-backend\ntime=2026-03-04T16:57:30.837Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-cluster-service\ntime=2026-03-04T16:57:31.015Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-maestro-to-db\ntime=2026-03-04T16:57:31.220Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-metrics\ntime=2026-03-04T16:57:31.380Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-nothing\ntime=2026-03-04T16:57:31.514Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=133.260993ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-nothing\ntime=2026-03-04T16:57:31.713Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=140.193665ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-nothing\ntime=2026-03-04T16:57:31.777Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=maestro name=allow-registration-job-to-maestro\ntime=2026-03-04T16:57:31.913Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=135.165595ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-registration-job-to-maestro\ntime=2026-03-04T16:57:32.113Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=139.121622ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1/namespaces/maestro/authorizationpolicies/allow-registration-job-to-maestro\ntime=2026-03-04T16:57:32.180Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=maestro name=default\ntime=2026-03-04T16:57:32.313Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy delay=133.271733ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j3604096-svc-u5wowc8e.hcp.westus3.azmk8s.io:443/apis/security.istio.io/v1beta1/namespaces/maestro/peerauthentications/default\ntime=2026-03-04T16:57:32.826Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=maestro name=maestro\ntime=2026-03-04T16:57:33.282Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=maestro name=maestro-server\ntime=2026-03-04T16:57:33.780Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-03-04T16:57:33.780Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T16:57:33.949Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy numVersions=2\ntime=2026-03-04T17:02:38.179Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T17:02:38.179Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T17:02:38.179Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T17:02:42.674Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy release=maestro-server namespace=maestro status=failed description=\"Upgrade \\\"maestro-server\\\" failed: resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[clientId:52268a48-3ba3-4eb7-bae0-330881d68626 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-prow-maestro-j3604096.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j3604096] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:3 requests:map[cpu:200m memory:512Mi] zoneCount:4] image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:062f0656-796b-42b2-99c8-7ecf6461937b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T17:02:42.676Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5xUwW7aQBS8+yuefAoSjiikiUSVQ5r2VLVCouohVQ/r3am9sN613j6DqPj4yqAUQkzi9Ga/nRnvm7EmcRBixNCwRqRbMkqUqNzhIuQLaPlivZlGYeuLIe1H31SFfyOs4HeTWCv9OB78TIiI0jl4ZTXutA6Nl3RIaaUQhcOTxz12Bo42Crz8CK6pcO+UrY5wmck7WJ9Qu7Cp8Jp6J66PZB2iFIyY5Qy1LJyKsYNzp3nWOPfRemN9seM1zmX54f2U0EgZ2P5RYoOfBWf1pkUp58I6y5Vewpu30rRrooCzuHf9rfRHU+SMLy+TIWx1pzUv0XyQ8j8MYhQ2Cu/OskXI2zufz34OzZAZh5U14PtnEXYQdgZ+Dd5K4OMfprUWfPYbr8meYLp9nj+LLyu41v2QJZSTUpfQy16Efqrn0+0Ady41A7iNFF6s3uXWggx+q8Y9sS359SFZNjnYQxA/t+0Sky2tSzBIbIUoqqoph6wBTxdGCdrxRToeja+z0SQbXX1/dz19fzOdTB7SAV1eUjfmZjoaT6/GD+lgkGxpEaynpfXm1noPPurE4OlQhccdeFp+yZZqDu3p4aJDqoPZg/uJDMmFYkgVYlRFKxnYgCnfHC1vEPXfAAAA//+FAnesuQUAAA==\"\ntime=2026-03-04T17:02:42.678Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-03-04T17:02:42.679Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T17:04:42.679Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: maestro, kind: Deployment, status: Failed\\nresource not ready, name: maestro-db, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96661e640a726b3a7f8212ebb7a4d8245515d5ea316dbcb7ed5ebd4169e02661","failure_pattern":"resource not ready, name: maestro-db, kind: Deployment, status: Failed","search_query":"resource not ready, name: maestro-db, kind: Deployment, status: Failed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["96661e640a726b3a7f8212ebb7a4d8245515d5ea316dbcb7ed5ebd4169e02661"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029214681754767360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"b60ce1f30ada0d35e621e79382aa3e59e0ec4f23","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T15:17:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"845351bb32de507480a14a425519b714de5e30cb219903687c49be013eee62d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029214681754767360","occurred_at":"2026-03-04T15:17:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T15:18:46.947Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T15:18:49.217Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T15:18:50.746Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d498fef569e9f8d26aae977c9ab8a615395f9512dd21737f12a5032062dc88f8\ntime=2026-03-04T15:20:51.729Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4767360/providers/Microsoft.Resources/deployments/d498fef569e9f8d26aae977c9ab8a615395f9512dd21737f12a5032062dc88f8/operationStatuses/08584289693556557198\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4767360/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4767360/providers/Microsoft.Monitor/accounts/hcps-j4767360/providers/Microsoft.Authorization/roleAssignments/af63ca07-014b-555e-9683-9c45578cfd0c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T151923Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'bcdc0dbb-84ee-42d8-8e8f-982ca44294f7', request correlation id '44d55cbe-c178-4702-84b1-23d37bc961db'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029214302019260416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4314,"pr_state":"closed","pr_sha":"35511ce9c20977aff9135f8686faa6039565f5ce","final_merged_sha":"35511ce9c20977aff9135f8686faa6039565f5ce","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T15:16:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029213130604679168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"5add497600e5bfec4a93b6e3bd8ffa357ecb01fc","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T15:11:25Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"34af6207698eb0b0bd80dce071c8256d9cc91d02ef8d3b8ee38cb87ef336a72b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029213130604679168","occurred_at":"2026-03-04T15:11:25Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T15:43:44.365Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T15:43:46.577Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T15:43:47.701Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ece3ee87dbeed2435e682c0664a8a2bc4d71b6fbd1ca26ce3ec380e7399db59c\ntime=2026-03-04T15:45:18.541Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4679168/providers/Microsoft.Resources/deployments/ece3ee87dbeed2435e682c0664a8a2bc4d71b6fbd1ca26ce3ec380e7399db59c/operationStatuses/08584289678584280771\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4679168/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4679168/providers/Microsoft.Monitor/accounts/hcps-j4679168/providers/Microsoft.Authorization/roleAssignments/f48158a1-781a-5e71-931d-510e32066f5b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T154404Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '45b42b6c-b21a-4396-8ee9-9ed0e9f95a88', request correlation id '6fe01212-66ae-426e-8fe0-a55d49d614b3'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029212884671664128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4154,"pr_state":"closed","pr_sha":"87ae44a893e18a3f688b7a042698f29e92dcdb8d","final_merged_sha":"87ae44a893e18a3f688b7a042698f29e92dcdb8d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T15:10:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4315/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029210791357452288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4315,"pr_state":"closed","pr_sha":"0b3a4559c5f1ad0b6eb59ba665c2228803c96004","final_merged_sha":"0b3a4559c5f1ad0b6eb59ba665c2228803c96004","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T15:02:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4316/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029208094692610048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4316,"pr_state":"closed","pr_sha":"27be21ced194f9cb3de8f9a2570574a567ed8a52","final_merged_sha":"a88a31f9140561442c3228d6941479d5c3b90d5f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T14:51:25Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5518d1cbaf7e67c915ce4de521dfd25a3963b61be682c16abfb8852fab757a9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4316/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029208094692610048","occurred_at":"2026-03-04T14:51:25Z","failed_at":"provision","signature_id":"ad99613dcbc1c246c95f1216d45ccdfdf152f6df0f9a1a2f04500834e88ea6db","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-04T15:40:58.991Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-04T15:40:59.568Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2610048.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2610048.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d databaseName:arohcpprow-rp-j2610048 databaseUrl:https://arohcpprow-rp-j2610048.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-1n42z7iz/pipeline@sha256:c0b52f4451b052ec0e26f17df6c414a1b8bd50fdf71ec8bed58bd55d2084ca5b replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-1n42z7iz/pipeline:pull workloadIdentityClientId:8b702cd7-2c4a-4e01-8dbe-6220f9c6da7f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T15:40:59.569Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:40:59.827Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-04T15:41:00.458Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-04T15:41:00.458Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:41:00.576Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:46:07.733Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T15:46:07.733Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:46:07.733Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:46:11.060Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2610048.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2610048.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d databaseName:arohcpprow-rp-j2610048 databaseUrl:https://arohcpprow-rp-j2610048.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-1n42z7iz/pipeline@sha256:c0b52f4451b052ec0e26f17df6c414a1b8bd50fdf71ec8bed58bd55d2084ca5b replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-1n42z7iz/pipeline:pull workloadIdentityClientId:8b702cd7-2c4a-4e01-8dbe-6220f9c6da7f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T15:46:11.061Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTwW4aMRC971eM9hSkdQRpmgNVDmnaU5UKNT2l6sFrv4KD17Mae0FEfHxlKGWFlpbrm/fG855nCo9EgsidGES6J6uTTrr2uOL6FSZ9ccFOYxIX5hXtoa+6wV8IK4QdElttDvDoR0FEVH5jj48uWBfmZUVlrc0SwSoPbSEKHiY5DrmkhdXCtGW1Fz5DVs6gV1EHcYMkzsQB0SOHX27+pNv+W/qtEyjpQnINlNlRLtWeZWdfGf6/kU9oPW8ahDTgZYD/YGTWeT8QWtt5r+ojfirs0oLFvek8yIy9M5sdy3teHzL7x7szQHILhOSMPjFzQfbPMII0E145C3n0Osb+8HH/n2qJjVrpzqfzf/7EwSWWy+L6o3kwhrt9xAPc4ueHYtnVkICE+DkvbCy2tF5AQHktYtJNSzXSGgh0ZXVChq/Km/HNnRq/U+Pb75P309vJdDx+KUd0fU1nOXfTyeSlHI2KLb2yC7R0wd67ECC9M+NAx+vqn9XpPRVbaoVz9ThoRS3bPfmyJhV5nlfUIEY9zy1ZLITqTc+8RTS/AwAA//+ODPq5DAQAAA==\"\ntime=2026-03-04T15:46:11.063Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:46:11.063Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T15:47:11.070Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-04T15:47:11.562Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2610048.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2610048.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d databaseName:arohcpprow-rp-j2610048 databaseUrl:https://arohcpprow-rp-j2610048.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-1n42z7iz/pipeline@sha256:c0b52f4451b052ec0e26f17df6c414a1b8bd50fdf71ec8bed58bd55d2084ca5b replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-1n42z7iz/pipeline:pull workloadIdentityClientId:8b702cd7-2c4a-4e01-8dbe-6220f9c6da7f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T15:47:11.562Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:47:11.906Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-04T15:47:12.199Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:47:12.199Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:47:12.472Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-04T15:47:13.465Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:47:13.465Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-04T15:47:13.828Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-04T15:47:14.268Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-04T15:47:14.389Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-04T15:47:14.837Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-04T15:47:15.334Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-04T15:47:15.456Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-04T15:47:15.582Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-04T15:47:15.834Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-04T15:47:16.225Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-04T15:47:16.542Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-04T15:47:16.896Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-04T15:47:17.237Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-04T15:47:17.237Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:47:17.352Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-04T15:52:20.165Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T15:52:20.165Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:52:20.165Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:52:24.029Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2610048.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2610048.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d databaseName:arohcpprow-rp-j2610048 databaseUrl:https://arohcpprow-rp-j2610048.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-1n42z7iz/pipeline@sha256:c0b52f4451b052ec0e26f17df6c414a1b8bd50fdf71ec8bed58bd55d2084ca5b replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-1n42z7iz/pipeline:pull workloadIdentityClientId:8b702cd7-2c4a-4e01-8dbe-6220f9c6da7f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T15:52:24.030Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5RTwW7aQBS8+yuefALJGxFKGokqB5r2VKVCbU+peljvTmHDep/1dg0i4uMrYyhWGlp6s2ZnxrPv7WQeiQSRGzGIdEdWJ5106THg8gkmfXLBTmMSFxYFddBnXeE3hDXCHom1Nkd4+D0jIsq/QtbOYGYMNyHlBeWlNisE235qYbU0dV503C/s0cIe2kIUPExyHM4w37tgXVj0LNW/hR9Qe95W6KIcDtX5SLMmLVncs2795uyd2e5Z3vNGVUjiTPyLfA5Ia4GQnNEvMh11R59X9IfpPXBwieWyyPccfrrFg677k9HPjUBJE5KroMyecqn2LPuQ7v+uNDMyb7x/ZXt1470qT/gffzOCNBdeOwu59zrGvjp2YdQKW7XWje8vOM9+vMtWTQkJSIgf2wcbsx1tlhBQO5KYdFVTibQBAg2sTmjhQT4ejd+q0Rs1mny7vplObqfXt4/5kK6u6BznZjwdTx7z4TDb0RO7QCsX7J0LAdKrGQc6tatfq5d9ynZUC7enp6AF1Ww78mUmBXleFFQhRr1oLVkshMpt7/IW0fwKAAD//6LgrtgMBAAA\"\ntime=2026-03-04T15:52:24.032Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:52:24.032Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T15:53:24.034Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-04T15:53:24.611Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2610048.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2610048.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d databaseName:arohcpprow-rp-j2610048 databaseUrl:https://arohcpprow-rp-j2610048.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-1n42z7iz/pipeline@sha256:c0b52f4451b052ec0e26f17df6c414a1b8bd50fdf71ec8bed58bd55d2084ca5b replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-1n42z7iz/pipeline:pull workloadIdentityClientId:8b702cd7-2c4a-4e01-8dbe-6220f9c6da7f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T15:53:24.611Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:53:24.932Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-04T15:53:25.199Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:53:25.199Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:53:25.424Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-04T15:53:26.454Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:53:26.454Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-04T15:53:26.794Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-04T15:53:27.143Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-04T15:53:27.320Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-04T15:53:27.774Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-04T15:53:28.074Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-04T15:53:28.198Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-04T15:53:28.341Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-04T15:53:28.679Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-04T15:53:29.007Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-04T15:53:29.386Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-04T15:53:29.733Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-04T15:53:30.024Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-04T15:53:30.024Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:53:30.141Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-04T15:58:32.984Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T15:58:32.984Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:58:32.984Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:58:37.241Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj2610048.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj2610048.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d databaseName:arohcpprow-rp-j2610048 databaseUrl:https://arohcpprow-rp-j2610048.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-1n42z7iz/pipeline@sha256:c0b52f4451b052ec0e26f17df6c414a1b8bd50fdf71ec8bed58bd55d2084ca5b replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-1n42z7iz/pipeline:pull workloadIdentityClientId:8b702cd7-2c4a-4e01-8dbe-6220f9c6da7f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:11ca5fcd-a6a4-4c6e-b5c2-8e0828528f7d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-04T15:58:37.242Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQW8aMRCF7/srRnsCaR3R0LQVVQ407alKhZqeUvXgtV/BwetZjW0QET++WihlhUDl+vw974zfvsIjkSByFoNI92R10knXHgOuX2DSVxfsJCZxYV7RXvqmG/yTsELYKbHV5iAPfxZEROV39igrKj20hSh4mOQ4dJIWVgvTltWefIKsnEHvRNXaLBGsapDEmXjG9Bmt502DkM74zvAzQKY5LRCSM/pkkiu+9wQjSDPhlbOQB6/jjjoY434HtcRGrXT26fKejxxcYrlu7L+eqTGc96teZh84/HbzR932B9OvWaAkh+QaKLNDrvVepLtsP7lgXZj3+f9HPTUyy96f8bbZe1Uf9VNjTgsW97pLbsbemc2O8p7Xh9TOPWPx62OxzDUkICF+6X7YWGxpvYCAuieJSTct1UhrINDA6oROHpS3o9t3ajRWo7c/3txN7saT8ei5HNLNDV1kPkzG75/L4bDY0gu7QEsX7L0LAdKrGQc6tqtfq9M+FVtqhbvT46AVtWz38HWXVOR5XlGDGPW8u5LFQqje9Ja3iOZPAAAA//9xl/54DAQAAA==\"\ntime=2026-03-04T15:58:37.246Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-04T15:58:37.246Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-04T15:59:37.252Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029206989376065536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T14:47:01Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ca8ae23743ff018b4628fefe335af05b552cc90750e748c3c6f7e427fae0fa12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029206989376065536","occurred_at":"2026-03-04T14:47:01Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T15:19:40.232Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T15:19:42.448Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T15:19:43.595Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9c9ff51ae6dd810e90c78e645c16a1e8f40cf267875768850d4424dac4a39dea\ntime=2026-03-04T15:21:44.461Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6065536/providers/Microsoft.Resources/deployments/9c9ff51ae6dd810e90c78e645c16a1e8f40cf267875768850d4424dac4a39dea/operationStatuses/08584289693025784838\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6065536/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6065536/providers/Microsoft.Monitor/accounts/services-j6065536/providers/Microsoft.Authorization/roleAssignments/c5a107d0-e083-5808-9a23-2e89a91f723e\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T151957Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6772a42f-418d-4914-b2b3-caa66795dd17', request correlation id '5825ae10-b7aa-4247-8373-e848ee48a8d9'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029202907387138048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4314,"pr_state":"closed","pr_sha":"e6e11ea9c3bcb53cac6c9f794d57a859c814053b","final_merged_sha":"35511ce9c20977aff9135f8686faa6039565f5ce","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T14:30:48Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"977c52174ea3118b799adb5a1c7de22125fccd5279cf6be349d3798c3b85c61f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029202907387138048","occurred_at":"2026-03-04T14:30:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029200330842968064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"b60ce1f30ada0d35e621e79382aa3e59e0ec4f23","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T14:20:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b596b40ffd57b3f2f4112c6d3d01cd4786acc7fab7ff72a2f91ccd199553aba2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029200330842968064","occurred_at":"2026-03-04T14:20:33Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T14:49:52.535Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T14:49:54.872Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T14:49:56.801Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=84f1fcf9b1b78d09b21ea787e1156344193598fddfa40eb59946d0f2e4d17c84\ntime=2026-03-04T14:51:57.853Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2968064/providers/Microsoft.Resources/deployments/84f1fcf9b1b78d09b21ea787e1156344193598fddfa40eb59946d0f2e4d17c84/operationStatuses/08584289710899079350\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2968064/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2968064/providers/Microsoft.Monitor/accounts/hcps-j2968064/providers/Microsoft.Authorization/roleAssignments/13ebece5-8654-5bfa-b79f-8789153eeb50\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T145021Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '509927fe-068a-42fa-9c8d-d4429109c379', request correlation id 'a5d00aa0-eccf-4134-a243-97d535dd85c3'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029200090807144448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4309,"pr_state":"closed","pr_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","final_merged_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T14:19:36Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1d35d73a61ab7d758dbe830fe08eea9b5dacbbf9ddb64f82941e1e0d66950645","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029200090807144448","occurred_at":"2026-03-04T14:19:36Z","failed_at":"e2e","signature_id":"82ed455a6b1797f9c915134d774016bf30b2d1f1201bdaebc1df384e0fcd3084","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000c24c80?, {0x7fcb2c1bad68?, 0xc000a45a70?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:603 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000c72240, {0x7fcb2c1bad68, 0xc000a45a70}, {0x781823f, 0x14}, {0xc000014109, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:367 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func2.2({0x7fcb2c1bad68, 0xc000a45a70})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_api.go:60 +0xf1\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc000a45a70?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 71\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402]: Test Panicked: failed to get subscription ID: AzureCLICredential: signal: killed\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000c24c80?, {0x7fcb2c1bad68?, 0xc000a45a70?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:603 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000c72240, {0x7fcb2c1bad68, 0xc000a45a70}, {0x781823f, 0x14}, {0xc000014109, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:367 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func2.2({0x7fcb2c1bad68, 0xc000a45a70})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_api.go:60 +0xf1\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc000a45a70?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 71\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402]: Test Panicked: failed to get subscription ID: AzureCLICredential: signal: killed\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402]: Test Panicked","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39c6c5ed08e8e90ee04db2b87d32b29b6f38089eb34d7f09ee21959da678ff0e","failure_pattern":"Test Panicked","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:402]: Test Panicked"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["39c6c5ed08e8e90ee04db2b87d32b29b6f38089eb34d7f09ee21959da678ff0e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029194263262138368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4205,"pr_state":"closed","pr_sha":"2cf31954feaa13843a9fd0eb12d86f2882046a70","final_merged_sha":"a36f37aae6ead63c15a10186a904f97e4e0fc50d","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T13:56:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029185211761168384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T13:20:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1622400d535845314cde39e00e4ad6ae6c9b6d96a976dd026fe2eb76233baebc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029185211761168384","occurred_at":"2026-03-04T13:20:29Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T13:21:40.266Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T13:21:42.503Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T13:21:43.711Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b0ef20b1e9322a8c9fe7ed039ecf1eff448586a1d470f852a0b87085ae907dd3\ntime=2026-03-04T13:23:44.583Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1168384/providers/Microsoft.Resources/deployments/b0ef20b1e9322a8c9fe7ed039ecf1eff448586a1d470f852a0b87085ae907dd3/operationStatuses/08584289763824543640\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1168384/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1168384/providers/Microsoft.Monitor/accounts/hcps-j1168384/providers/Microsoft.Authorization/roleAssignments/bbd32259-8b0e-5394-a94e-abe3679a5071\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T132205Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'd319f7d5-b87c-4f1a-91f4-62d0f4db254e', request correlation id '7d61af07-ab05-42c3-83aa-6cde1db6d789'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029173309312077824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4154,"pr_state":"closed","pr_sha":"59174f27d02916fe85b4c6242c40f20020c23b9c","final_merged_sha":"87ae44a893e18a3f688b7a042698f29e92dcdb8d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T12:33:11Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9c16bf28a0770b21f779b4eafe9fdbd8c59e9aca0c3950b556e3cc6b3433fa9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029173309312077824","occurred_at":"2026-03-04T12:33:11Z","failed_at":"e2e","signature_id":"f92b5e347e30dea742f8d6c7a13715810e85ad89ab63fc6ba5aaef9205cfce47","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:210]: Unexpected error:\n \u003c*errors.joinError | 0xc000a5a630\u003e: \n invalid VM size test error: PUT https://rp.j2077824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-pmfgpl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Forbidden: field is immutable\",\n \"target\": \"properties.platform\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Forbidden: field is immutable\",\n \"target\": \"properties.platform.vmSize\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc0015986c0\u003e{\n msg: \"invalid VM size test error: PUT https://rp.j2077824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-pmfgpl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001589e30\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"d3e1ae62-af6f-4f56-9ed6-d0e37c1e8237\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 13:53:38 GMT\",\n ],\n \"Content-Length\": [\"515\"],\n \"X-Envoy-Upstream-Service-Time\": [\"131\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0015985e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 111, 114, 98, 105, 100, 100, 101, 110, 58, 32, 102, 105, 101, 108, 100, 32, 105, 115, 32, 105, 109, 109, 117, 116, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 112, 108, 97, 116, 102, 111, 114, 109, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:210]: Unexpected error:\n \u003c*errors.joinError | 0xc000a5a630\u003e: \n invalid VM size test error: PUT https://rp.j2077824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-pmfgpl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Forbidden: field is immutable\",\n \"target\": \"properties.platform\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Forbidden: field is immutable\",\n \"target\": \"properties.platform.vmSize\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc0015986c0\u003e{\n msg: \"invalid VM size test error: PUT https://rp.j2077824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-pmfgpl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"2 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001589e30\u003e{\n ErrorCode: \"MultipleErrorsOccurred\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"MultipleErrorsOccurred\",\n ],\n \"X-Ms-Request-Id\": [\n \"d3e1ae62-af6f-4f56-9ed6-d0e37c1e8237\",\n ],\n \"Date\": [\n \"Wed, 04 Mar 2026 13:53:38 GMT\",\n ],\n \"Content-Length\": [\"515\"],\n \"X-Envoy-Upstream-Service-Time\": [\"131\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0015985e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 50, 32, 101, 114, 114, 111, 114, 115, 32, 111, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 111, 114, 98, 105, 100, 100, 101, 110, 58, 32, 102, 105, 101, 108, 100, 32, 105, 115, 32, 105, 109, 109, 117, 116, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 112, 108, 97, 116, 102, 111, 114, 109, 34, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4fc7eff73bf2ec16dfa40270f65306b3452cd0e69caa96209df9bd3840e092d9","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Forbidden: field is immutable; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4fc7eff73bf2ec16dfa40270f65306b3452cd0e69caa96209df9bd3840e092d9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029173065857896448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T12:32:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f11ea32ad98f4dc1a00fbf99c0c069ce215d903ae677822446910fc381bc27bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029173065857896448","occurred_at":"2026-03-04T12:32:13Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T12:59:17.588Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T12:59:19.482Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T12:59:20.587Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cb14cf7492012f6e7f29e58cb3f6b6c850e54d3a78f7e0020964ae4d4544f68b\ntime=2026-03-04T13:01:21.445Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7896448/providers/Microsoft.Resources/deployments/cb14cf7492012f6e7f29e58cb3f6b6c850e54d3a78f7e0020964ae4d4544f68b/operationStatuses/08584289777255371328\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7896448/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7896448/providers/Microsoft.Monitor/accounts/services-j7896448/providers/Microsoft.Authorization/roleAssignments/48c66f00-12ec-5fee-8c21-59862a32ffbd\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T125943Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e95038bf-e92c-4f31-bb62-6068506f3876', request correlation id '267d2c1b-8f93-4a0a-a28d-1cb57d6ae07c'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7896448/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7896448/providers/Microsoft.Monitor/accounts/hcps-j7896448/providers/Microsoft.Authorization/roleAssignments/10b99d6e-338a-5355-afce-a92eca48ebbd\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T125942Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c4fa6d90-9d8c-4341-8c06-a2a9bb96ca2a', request correlation id '267d2c1b-8f93-4a0a-a28d-1cb57d6ae07c'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029173020337115136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T12:32:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029172029575401472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4303,"pr_state":"closed","pr_sha":"7148a34a98c3daf4639ce203a1af6126324406ca","final_merged_sha":"7148a34a98c3daf4639ce203a1af6126324406ca","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T12:28:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"46c6ca0bb6cd5a4fc0f9bc874704950851b0189c0fb3835618ecbdf6333c10bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029172029575401472","occurred_at":"2026-03-04T12:28:06Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T12:58:02.547Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T12:58:04.497Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T12:58:05.699Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b7608decf4d7e86e43b8ad772dc997e58019e2b2d7f6d61fad459af928bcb03d\ntime=2026-03-04T13:00:06.433Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5401472/providers/Microsoft.Resources/deployments/b7608decf4d7e86e43b8ad772dc997e58019e2b2d7f6d61fad459af928bcb03d/operationStatuses/08584289778004909795\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5401472/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5401472/providers/Microsoft.Monitor/accounts/hcps-j5401472/providers/Microsoft.Authorization/roleAssignments/a271e06e-5b1b-5d06-91ca-2b93e55309e3\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T125826Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7f7ac6f5-3c2d-4c73-b3c7-e32dba6f02a7', request correlation id 'c780276f-29b9-416a-afcc-859b0b146c24'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029167137750257664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4309,"pr_state":"closed","pr_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","final_merged_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T12:08:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3696/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029157323741597696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3696,"pr_state":"open","pr_sha":"39008467af52385dd11f0ef4cc894e5bc149fab8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T11:29:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029154545187164160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T11:18:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3f27dd38f1af1ec57b4bc6773f56a0c8ccbf6d98abeca9181db173fe15fb95b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029154545187164160","occurred_at":"2026-03-04T11:18:37Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T11:53:00.963Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T11:53:03.044Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T11:53:04.184Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=32bf687ec575978e9034d135870fdf361b601dc0ea10f169720c05d939c188df\ntime=2026-03-04T11:55:05.227Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7164160/providers/Microsoft.Resources/deployments/32bf687ec575978e9034d135870fdf361b601dc0ea10f169720c05d939c188df/operationStatuses/08584289817019116868\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7164160/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7164160/providers/Microsoft.Monitor/accounts/services-j7164160/providers/Microsoft.Authorization/roleAssignments/4bc99e86-4cd0-5209-82ab-b97d7371f03e\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T115349Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0ca62049-817f-469b-8e30-9b5170f94808', request correlation id '287a5bea-bfb7-4d55-99c4-06bb805450d2'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029153795467907072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T11:15:39Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c608dd0823d60d5d929146bbbc8bf3a46d56d28fbb2c60c1710c538902aae2a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029153795467907072","occurred_at":"2026-03-04T11:15:39Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T11:49:19.351Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T11:49:22.079Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T11:49:23.388Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d60f8e60b9a20bcae04f5f163622acd095ae978be178d068a7b8719fa6586c2a\ntime=2026-03-04T11:51:24.456Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7907072/providers/Microsoft.Resources/deployments/d60f8e60b9a20bcae04f5f163622acd095ae978be178d068a7b8719fa6586c2a/operationStatuses/08584289819228353110\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7907072/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7907072/providers/Microsoft.Monitor/accounts/hcps-j7907072/providers/Microsoft.Authorization/roleAssignments/fc97917e-239a-519a-9fa3-5a6ae6fd8f6c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T114939Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '4c802878-9ab7-4ca4-9e5e-ac80a0dcadd5', request correlation id '33bf18dd-afdd-4352-a41e-935583b05a05'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7907072/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7907072/providers/Microsoft.Monitor/accounts/services-j7907072/providers/Microsoft.Authorization/roleAssignments/d96906f2-df7b-519a-8c8f-7d3a4044ae9e\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T114945Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f10484e9-9811-4433-8f73-509048286ff8', request correlation id '33bf18dd-afdd-4352-a41e-935583b05a05'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029152794845384704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4154,"pr_state":"closed","pr_sha":"59174f27d02916fe85b4c6242c40f20020c23b9c","final_merged_sha":"87ae44a893e18a3f688b7a042698f29e92dcdb8d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T11:11:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"be1f4e57fe6ffedc48f3b430b0137ac014955d62eb0aa99a0a069d24e1ec0793","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029152794845384704","occurred_at":"2026-03-04T11:11:40Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T11:49:23.487Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T11:49:25.491Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T11:49:26.819Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f1fd91dcc9b64f164c15a549c41f4610b624a6cac707401d6b57805d5c8165f4\ntime=2026-03-04T11:51:27.880Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5384704/providers/Microsoft.Resources/deployments/f1fd91dcc9b64f164c15a549c41f4610b624a6cac707401d6b57805d5c8165f4/operationStatuses/08584289819194411977\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5384704/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5384704/providers/Microsoft.Monitor/accounts/hcps-j5384704/providers/Microsoft.Authorization/roleAssignments/5d4955cf-43b7-562d-a752-eb6993861d8f\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T114941Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '307464a7-269c-462b-bf87-b3ce6fdd6e49', request correlation id '3203dd2c-29bf-4e66-a0c9-7607277718c8'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029151404471357440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T11:06:09Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c10c05a0d66fbd681934afb325374ca7095a3573821f384d128537b2f2c8a643","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029151404471357440","occurred_at":"2026-03-04T11:06:09Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T11:35:27.488Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T11:35:30.012Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T11:35:31.421Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b540db4c2c415358f1ae9b1ef6f5fedf81a6c8a32a2f2b2563db748eb861cd19\ntime=2026-03-04T11:37:02.531Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1357440/providers/Microsoft.Resources/deployments/b540db4c2c415358f1ae9b1ef6f5fedf81a6c8a32a2f2b2563db748eb861cd19/operationStatuses/08584289827547348911\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1357440/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1357440/providers/Microsoft.Monitor/accounts/services-j1357440/providers/Microsoft.Authorization/roleAssignments/39d620df-9b16-51d5-a208-70f2672b162c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T113554Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '53d127b6-a5cd-4234-8664-a74d870ec7c0', request correlation id '42097278-847f-4b3e-99ab-626a4e16016b'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029150655028924416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T11:03:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029149860279619584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T11:00:01Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"97e368a8893cd169fff5b7684b0045ef7f7758a30be9e1fe8a1f85b6bbaa419b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029149860279619584","occurred_at":"2026-03-04T11:00:01Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T11:37:48.994Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T11:37:52.405Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T11:37:53.534Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=76406f78b697f196105df3a716a0c7bff6e645ad76711fdcdecedba00afe6d73\ntime=2026-03-04T11:39:54.583Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9619584/providers/Microsoft.Resources/deployments/76406f78b697f196105df3a716a0c7bff6e645ad76711fdcdecedba00afe6d73/operationStatuses/08584289826126209316\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9619584/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9619584/providers/Microsoft.Monitor/accounts/hcps-j9619584/providers/Microsoft.Authorization/roleAssignments/08548882-11d3-5865-8fa8-9560d5772499\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T113817Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '68c6e2ab-212b-472f-b26e-de99e69a4d4c', request correlation id '5a8b2373-15ae-480c-9e15-0574d4c90dc0'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029148737439272960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4154,"pr_state":"closed","pr_sha":"59174f27d02916fe85b4c6242c40f20020c23b9c","final_merged_sha":"87ae44a893e18a3f688b7a042698f29e92dcdb8d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T10:55:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"89d8c82df84e2be621c9f9add6628b5b8d94791d8eeff0197e76a2be25e97e1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029148737439272960","occurred_at":"2026-03-04T10:55:33Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T11:01:00.465Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T11:01:03.158Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T11:01:04.358Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ea3a1d64c5c8a1a4baa898bd8e799908a9f8ad60c74d7f71957550080033c73b\ntime=2026-03-04T11:03:06.143Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9272960/providers/Microsoft.Resources/deployments/ea3a1d64c5c8a1a4baa898bd8e799908a9f8ad60c74d7f71957550080033c73b/operationStatuses/08584289848218121125\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9272960/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9272960/providers/Microsoft.Monitor/accounts/services-j9272960/providers/Microsoft.Authorization/roleAssignments/e67c5551-228f-545b-9aa8-193e6153e2b8\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T110121Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '30765463-f0b2-4f74-b998-e117d0e67a36', request correlation id '9328c495-d1b1-48e7-9af6-f2abf1bf0d71'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4313,"pr_state":"closed","pr_sha":"2441e9f6440cdc27788b30096a088844f016b47a","final_merged_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T09:58:34Z"},"failed_at":["e2e"],"failed_test_count":18,"occurrences":[{"row_id":"1192afe3b200afe28ef92e12346d822d5013e802f9a753aa3c96934ea6f75777","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"946255afb135c782a7b77a91543862fe47b7caa4b964adbf0a697ef7585c1fa8","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f31a70\u003e: \n failed to create HCP cluster list-test-cluster-7wr5z5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: failed waiting for cluster=\"list-test-cluster-7wr5z5\" in resourcegroup=\"cluster-listing-85q47f\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-7wr5z5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: failed waiting for cluster=\\\"list-test-cluster-7wr5z5\\\" in resourcegroup=\\\"cluster-listing-85q47f\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001076110\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f\",\n },\n \u003c*fmt.wrapErrors | 0xc000f31a40\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-7wr5z5\\\" in resourcegroup=\\\"cluster-listing-85q47f\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001076110\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f31a70\u003e: \n failed to create HCP cluster list-test-cluster-7wr5z5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: failed waiting for cluster=\"list-test-cluster-7wr5z5\" in resourcegroup=\"cluster-listing-85q47f\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-7wr5z5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: failed waiting for cluster=\\\"list-test-cluster-7wr5z5\\\" in resourcegroup=\\\"cluster-listing-85q47f\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001076110\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f\",\n },\n \u003c*fmt.wrapErrors | 0xc000f31a40\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-7wr5z5\\\" in resourcegroup=\\\"cluster-listing-85q47f\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001076110\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-7wr5z5 in resource group cluster-listing-85q47f\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"138427e0bf47d3c5942e22b9c62d0be1d262b7349ba18f912ab473c773712edf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"e8ce78518a553246533032800ed81392d08d152b8f4de5703748467befafbaa8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010395c0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-n9pws6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-n9pws6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00100c970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6\",\n },\n \u003c*fmt.wrapErrors | 0xc001039590\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-n9pws6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00100c970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010395c0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-n9pws6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-n9pws6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00100c970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6\",\n },\n \u003c*fmt.wrapErrors | 0xc001039590\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-n9pws6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00100c970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-n9pws6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"29d453de3956f37c0d375c0166855052b980b174a755be8efe906673466aba65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"96fa233040984537a92a69af289c5218713f756f1919b3498229d04fedc3f8f1","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00149b290\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-94fb29\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-94fb29\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ce09a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29\",\n },\n \u003c*fmt.wrapErrors | 0xc00149b260\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-94fb29\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ce09a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00149b290\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-94fb29\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-94fb29\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ce09a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29\",\n },\n \u003c*fmt.wrapErrors | 0xc00149b260\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-94fb29\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ce09a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-94fb29\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"3cccc3a00d4cf82967a9ae7ecc56c15c2f1c87c810f44599d782611b92fc5d0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"4d53f83fe8543893a9b52b8b1881d8e6b0efeb24c58ea474b88fca19758f37eb","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00178b0b0\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8grqgl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8grqgl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ec38b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl\",\n },\n \u003c*fmt.wrapErrors | 0xc000ef8ea0\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8grqgl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ec38b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00178b0b0\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8grqgl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8grqgl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ec38b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl\",\n },\n \u003c*fmt.wrapErrors | 0xc000ef8ea0\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-8grqgl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ec38b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-8grqgl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"5382155a8c040eaeabb31c40216463bdb6ce2268ff1a688df9678e96a91ff1b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"af77787d6a6d196225e9ed7c8f6bd6d83cb894c564f8b4fda9e12f9e97c67bda","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012cd380\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-kks6bc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-kks6bc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00131f040\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc\",\n },\n \u003c*fmt.wrapErrors | 0xc0012cd350\u003e{\n msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-kks6bc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00131f040\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012cd380\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-kks6bc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-kks6bc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00131f040\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc\",\n },\n \u003c*fmt.wrapErrors | 0xc0012cd350\u003e{\n msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-kks6bc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00131f040\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-kks6bc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"70387b202bd45b27fa78d51de145d380019117c95f5e308aa90663284a926db7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"35fc6764570070bed6d4b174b239fcdcc4e2d2cdcdd2dee6e1c2ece2e36c28e8","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001356660\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-w7zqgp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-w7zqgp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00132d850\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp\",\n },\n \u003c*fmt.wrapErrors | 0xc0013565d0\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-w7zqgp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00132d850\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001356660\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-w7zqgp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-w7zqgp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00132d850\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp\",\n },\n \u003c*fmt.wrapErrors | 0xc0013565d0\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-w7zqgp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00132d850\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-w7zqgp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"72d38d3c9cc1d6df6b6323c827bd44154e6270259cec639b1efd62605a7a525f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"0c3f688a522cc58d176e75acae7e1ad733382dd6c8cf13f4b6c279e02d95765d","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00038aed0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-blmgc7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001065e40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7\",\n },\n \u003c*fmt.wrapErrors | 0xc00038aea0\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-blmgc7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001065e40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00038aed0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-blmgc7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001065e40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7\",\n },\n \u003c*fmt.wrapErrors | 0xc00038aea0\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-blmgc7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001065e40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"897c7e78baa31e104396bf0aa79fd54622bbc321eaa2eb55959f600372016be7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"b0180976ac26fc5c3939ecf1e5fee78abeed7b44dbc049100ea6e82c7b006a63","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00159e8d0\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-cjg6hw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cjg6hw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000efae40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw\",\n },\n \u003c*fmt.wrapErrors | 0xc00159e8a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cjg6hw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000efae40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00159e8d0\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-cjg6hw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cjg6hw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000efae40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw\",\n },\n \u003c*fmt.wrapErrors | 0xc00159e8a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-cjg6hw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000efae40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-cjg6hw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"a1d471690b3e6ddd6b9bac170daf3626a4d2bef5cc7087328b855bcf369c3230","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"701ce9a67a3845fa98ec22c61512ca31a08d2c7e56dd8eaa1034666d55081610","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7fbe2ee580b0, 0xc0008f0ea0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc0008f0ea0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 12:20:17.181\n \"ts\"=\"2026-03-04 12:20:17.181376\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 12:20:17.181449\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-r8299g\" \"resourceGroup\"=\"customer-rg-fmprdk\"\n \"ts\"=\"2026-03-04 12:20:17.754133\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-r8299g\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 12:20:18.618579\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-gt5nj2\" \"resourceGroup\"=\"customer-rg-fmprdk\"\n \"ts\"=\"2026-03-04 12:20:18.844790\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-gt5nj2\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 12:20:19.791630\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 12:20:19.791746\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 12:20:19.791815\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 12:20:19.792915\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-108\"\n \"ts\"=\"2026-03-04 12:20:19.792942\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-124\"\n \"ts\"=\"2026-03-04 12:20:19.792927\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-136\"\n \"ts\"=\"2026-03-04 12:20:19.792966\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-fmprdk\"\n \"ts\"=\"2026-03-04 12:20:20.465045\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-108\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 12:20:20.752891\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-124\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 12:20:21.092977\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-136\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 12:20:21.529002\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 12:20:21.896103\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-108\" \"deployment\"=\"msi-deployment-108\"\n \"ts\"=\"2026-03-04 12:20:22.262610\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-124\" \"deployment\"=\"msi-deployment-124\"\n \"ts\"=\"2026-03-04 12:20:22.786160\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-136\" \"deployment\"=\"msi-deployment-136\"\n \"ts\"=\"2026-03-04 12:20:22.976200\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-gt5nj2\"\n \"ts\"=\"2026-03-04 12:20:24.393749\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 12:20:24.855086\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"customer-infra-basic-hcp-cluster-r8299g\"\n \"ts\"=\"2026-03-04 12:20:25.388961\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 12:20:25.389089\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 12:20:25.389193\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-fmprdk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7fbe2ee580b0, 0xc0008f0ea0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc0008f0ea0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 12:20:17.181\n \"ts\"=\"2026-03-04 12:20:17.181376\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 12:20:17.181449\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-r8299g\" \"resourceGroup\"=\"customer-rg-fmprdk\"\n \"ts\"=\"2026-03-04 12:20:17.754133\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-r8299g\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 12:20:18.618579\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-gt5nj2\" \"resourceGroup\"=\"customer-rg-fmprdk\"\n \"ts\"=\"2026-03-04 12:20:18.844790\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-gt5nj2\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 12:20:19.791630\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 12:20:19.791746\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 12:20:19.791815\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 12:20:19.792915\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-108\"\n \"ts\"=\"2026-03-04 12:20:19.792942\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-124\"\n \"ts\"=\"2026-03-04 12:20:19.792927\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-136\"\n \"ts\"=\"2026-03-04 12:20:19.792966\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-fmprdk\"\n \"ts\"=\"2026-03-04 12:20:20.465045\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-108\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 12:20:20.752891\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-124\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 12:20:21.092977\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-136\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 12:20:21.529002\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 12:20:21.896103\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-108\" \"deployment\"=\"msi-deployment-108\"\n \"ts\"=\"2026-03-04 12:20:22.262610\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-124\" \"deployment\"=\"msi-deployment-124\"\n \"ts\"=\"2026-03-04 12:20:22.786160\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-136\" \"deployment\"=\"msi-deployment-136\"\n \"ts\"=\"2026-03-04 12:20:22.976200\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-gt5nj2\"\n \"ts\"=\"2026-03-04 12:20:24.393749\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 12:20:24.855086\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-fmprdk\" \"deployment\"=\"customer-infra-basic-hcp-cluster-r8299g\"\n \"ts\"=\"2026-03-04 12:20:25.388961\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 12:20:25.389089\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 12:20:25.389193\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-fmprdk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"ab10a2843c89b737380d7b671a3972c79992eb26e0770420a8ca5c60db8da2e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"f4d018e9878fa8dc9b8852d3bb5f6d34625c6bc514ecbf12602353e9257fc41a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a29950\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-gdsz9k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-gdsz9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001136530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k\",\n },\n \u003c*fmt.wrapErrors | 0xc000a29920\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-gdsz9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001136530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a29950\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-gdsz9k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-gdsz9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001136530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k\",\n },\n \u003c*fmt.wrapErrors | 0xc000a29920\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-gdsz9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001136530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-gdsz9k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"ae780912117927ecf436acca5b094e174dd3b8e27a85d131e42329f3309b3237","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"908a5050690ff9c2e63d9caf5998d5f08260744307547e96e0edb30ceb20c7b7","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001585c80\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-t2t7rc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-t2t7rc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7f300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc\",\n },\n \u003c*fmt.wrapErrors | 0xc001585c50\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-t2t7rc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7f300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001585c80\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-t2t7rc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-t2t7rc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7f300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc\",\n },\n \u003c*fmt.wrapErrors | 0xc001585c50\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-t2t7rc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7f300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-t2t7rc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"b12794b717b7c83394c5871dc1118d5a3c425a31e87b45ffb8325ee76c7f75ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"350746bb0b3f9d95b4c8ce5d9871d087a018be557450c004e6ba2a8f22e3d702","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001289290\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-vftvd8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vftvd8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bc570\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8\",\n },\n \u003c*fmt.wrapErrors | 0xc001289260\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vftvd8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bc570\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001289290\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-vftvd8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vftvd8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bc570\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8\",\n },\n \u003c*fmt.wrapErrors | 0xc001289260\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vftvd8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bc570\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-vftvd8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"c9b0bdd92bab4388323a78355b521160772227f4cc1b59aa39e169ef866a882a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"120b4ca498df420bafa75a52df0c2ed51ab121b305364c1030be16eefaa8dbf1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d0f00\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ww94ws\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ww94ws\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011bc520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws\",\n },\n \u003c*fmt.wrapErrors | 0xc0010d0ed0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ww94ws\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011bc520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d0f00\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ww94ws\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ww94ws\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011bc520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws\",\n },\n \u003c*fmt.wrapErrors | 0xc0010d0ed0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ww94ws\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011bc520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ww94ws\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"ca05429e69028441fd2fe6d1a2f22beec49fa559ea7dd17dc4278791cf9866b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"c346eb097df02e6dfcb470a519e2ef41b98bb3dc8915d4065d895bdef07e564a","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010f49f0\u003e: \n failed to create HCP cluster ea-listsf9cdz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: failed waiting for cluster=\"ea-listsf9cdz\" in resourcegroup=\"ea-list-8r6rmq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listsf9cdz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: failed waiting for cluster=\\\"ea-listsf9cdz\\\" in resourcegroup=\\\"ea-list-8r6rmq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc9c10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq\",\n },\n \u003c*fmt.wrapErrors | 0xc0010f49c0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listsf9cdz\\\" in resourcegroup=\\\"ea-list-8r6rmq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc9c10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010f49f0\u003e: \n failed to create HCP cluster ea-listsf9cdz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: failed waiting for cluster=\"ea-listsf9cdz\" in resourcegroup=\"ea-list-8r6rmq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listsf9cdz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: failed waiting for cluster=\\\"ea-listsf9cdz\\\" in resourcegroup=\\\"ea-list-8r6rmq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc9c10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq\",\n },\n \u003c*fmt.wrapErrors | 0xc0010f49c0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listsf9cdz\\\" in resourcegroup=\\\"ea-list-8r6rmq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc9c10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listsf9cdz in resource group ea-list-8r6rmq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"f8d20db68ba639f7d3760612f6e22c6c8dd16efcacdc1743b70439331e260db8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"f9dd284ff80bcf092d903aa3149116a496f466fae6be8e9bfe91d2162392cd37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"f7d21b9b4e6186b9fe5e4535d9ef048d79a31e411f5fc5ff76d10eff5c5f2dc3","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001480c00\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lp4tkb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lp4tkb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f49a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb\",\n },\n \u003c*fmt.wrapErrors | 0xc001480bd0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lp4tkb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f49a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001480c00\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lp4tkb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lp4tkb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f49a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb\",\n },\n \u003c*fmt.wrapErrors | 0xc001480bd0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lp4tkb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f49a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-lp4tkb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"fa233934a3ea23684bf9720b8fe9a6d811c5afc4941aa1deeab58ce4456ab5cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"2114c584a92a1441d7c34c815e8ed50c5356ee3ebf4874d0bedfa7283d43dd85","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a656e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nnns7t\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nnns7t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00110f8a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t\",\n },\n \u003c*fmt.wrapErrors | 0xc000a656b0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nnns7t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00110f8a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a656e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nnns7t\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nnns7t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00110f8a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t\",\n },\n \u003c*fmt.wrapErrors | 0xc000a656b0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-nnns7t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00110f8a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-nnns7t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}},{"row_id":"fa7fae562158b88eb482e8ca6e61f058580c9d02657cb2294c0d9f74797bdbe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029134399739531264","occurred_at":"2026-03-04T09:58:34Z","failed_at":"e2e","signature_id":"7aa20e2377b021a7dd029583394bf39833d3b835f67e114002bd7de4be434aa1","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00151f380\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ffhbdr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ffhbdr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e7a020\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr\",\n },\n \u003c*fmt.wrapErrors | 0xc00151f350\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ffhbdr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e7a020\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00151f380\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ffhbdr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ffhbdr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e7a020\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr\",\n },\n \u003c*fmt.wrapErrors | 0xc00151f350\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ffhbdr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e7a020\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-ffhbdr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-blmgc7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-blmgc7, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029130601595932672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4310,"pr_state":"closed","pr_sha":"4b5ae236d9d72c2b4c53c9d7ce5e9b67247cf9bc","final_merged_sha":"4b5ae236d9d72c2b4c53c9d7ce5e9b67247cf9bc","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T09:43:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"96df0dd8d9297e6891e31f680b81f8ef284de374f4e16bd5f2831d88730c16a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029130601595932672","occurred_at":"2026-03-04T09:43:29Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-04T10:30:13.969Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-04T10:30:14.687Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-04T10:30:14.687Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:30:14.748Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-04T10:30:14.867Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-04T10:30:14.867Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:30:14.981Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:35:41.197Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed post-install: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T10:35:41.197Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:35:41.198Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:35:45.724Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config release=mce-config namespace=multicluster-engine status=failed description=\"Release \\\"mce-config\\\" failed: failed post-install: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-04T10:35:45.726Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yVT2/aTBDG7/4UI59AsiNCyHvwqxxo2lPUVmp6StXDencwG9Yz1u4CQuLDV4tt7EIoBOXA5WGe386fXU9k0INFx0sr0cEDKOGFF7nBAeevKP2TJpU5bzUVCdTSN1HiXsIV0k5xlZCtPPwVAQDEj2bpPNofbDBOIOYKKZW1lpaCRIElks8WtWTQp0IpplQyecvGoE1X4+CMk3cBCyuzRrZN6BWEio2Wmyajs7RPmpSm4sPrvIB7UG7eOa7nnSj+DXbbysLKNPxqZ1Nh27Byabxuj0IqNOEp/wWWz1gZ3oRcQ+D5tl6MOsq/Q13LqSxXohCezxGmSn2nDvPINNO7RteZzNl5VCGhRv4Hqhn31/1Id+xgqu1tbn9Xd95d8AotCZLYEmZWlLhmu+gDntGutMSplLz8gBEd447G5MRVjLO2gwdz8or3nsX7aZcDnvZtnIajuytiWArTvtM3hEN75ywM58L0p1fPXTW34Bl33ZpvKrRurme+96Fo+W5nj37/Hy2WOVpCj+5L2Asu2sJ6jhbB6xKdF2UFOfo1IsFACY9BHsTj0fi/dHSXjiY/b0fZ3Si7nbzEQ7i5gZMx99nk/iUeDqMtvLImWGhSD5oIbW+bMUG3xPrb63BtRVuoLId/u0QTqFjVwZdBEjBcJFCic6IISLYKLeSbXvEKnfwTAAD///fnqy1zBwAA\"\ntime=2026-03-04T10:35:45.727Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:35:45.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config step=deploy-mce-config error=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T10:36:45.732Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-04T10:36:51.428Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-04T10:36:51.428Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:36:51.929Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-04T10:36:52.104Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:36:52.104Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:36:52.275Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config numVersions=1\ntime=2026-03-04T10:36:53.274Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:36:53.274Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"/v1, Kind=ServiceAccount\" namespace=multicluster-engine name=klusterlet-addon-controller-v2\ntime=2026-03-04T10:36:53.594Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"/v1, Kind=ServiceAccount\" namespace=multicluster-engine name=grc-policy-addon-sa\ntime=2026-03-04T10:36:53.760Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"/v1, Kind=ServiceAccount\" namespace=multicluster-engine name=grc-sa\ntime=2026-03-04T10:36:53.928Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=open-cluster-management:klusterlet-addon-controller-v2\ntime=2026-03-04T10:36:54.414Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=open-cluster-management:grc:clusterrole\ntime=2026-03-04T10:36:54.537Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=open-cluster-management:grc:policyaddon-clusterrole\ntime=2026-03-04T10:36:54.659Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=open-cluster-management:klusterlet-addon-controller-v2\ntime=2026-03-04T10:36:54.966Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=multicluster-engine name=open-cluster-management:grc:clusterrolebinding\ntime=2026-03-04T10:36:55.206Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=open-cluster-management:grc:policyaddon-clusterrolebinding\ntime=2026-03-04T10:36:55.324Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=multicluster-engine name=grc-grc-policy-addon-role\ntime=2026-03-04T10:36:55.771Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=multicluster-engine name=grc-grc-role\ntime=2026-03-04T10:36:55.889Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=multicluster-engine name=grc-grc-policy-addon-rolebinding\ntime=2026-03-04T10:36:56.256Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=multicluster-engine name=grc-grc-rolebinding\ntime=2026-03-04T10:36:56.380Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"apps/v1, Kind=Deployment\" namespace=multicluster-engine name=klusterlet-addon-controller-v2\ntime=2026-03-04T10:36:56.477Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=96.223278ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/klusterlet-addon-controller-v2\ntime=2026-03-04T10:36:56.677Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=141.295013ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/klusterlet-addon-controller-v2\ntime=2026-03-04T10:36:56.975Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"apps/v1, Kind=Deployment\" namespace=multicluster-engine name=grc-policy-addon-controller\ntime=2026-03-04T10:36:57.141Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"apps/v1, Kind=Deployment\" namespace=multicluster-engine name=grc-policy-propagator\ntime=2026-03-04T10:36:57.277Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=135.890681ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/grc-policy-propagator\ntime=2026-03-04T10:36:57.477Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=141.283962ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/grc-policy-propagator\ntime=2026-03-04T10:36:57.543Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"addon.open-cluster-management.io/v1alpha1, Kind=AddOnDeploymentConfig\" namespace=multicluster-engine name=addon-hosted-config\ntime=2026-03-04T10:36:57.677Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=133.297092ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/namespaces/multicluster-engine/addondeploymentconfigs/addon-hosted-config\ntime=2026-03-04T10:36:57.877Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=141.285455ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/namespaces/multicluster-engine/addondeploymentconfigs/addon-hosted-config\ntime=2026-03-04T10:36:58.110Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"addon.open-cluster-management.io/v1alpha1, Kind=ClusterManagementAddOn\" namespace=\"\" name=config-policy-controller\ntime=2026-03-04T10:36:58.277Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=108.283574ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/clustermanagementaddons/config-policy-controller\ntime=2026-03-04T10:36:58.512Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"addon.open-cluster-management.io/v1alpha1, Kind=ClusterManagementAddOn\" namespace=\"\" name=governance-policy-framework\ntime=2026-03-04T10:36:58.677Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=105.252017ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/clustermanagementaddons/governance-policy-framework\ntime=2026-03-04T10:36:58.740Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"agent.open-cluster-management.io/v1, Kind=KlusterletAddonConfig\" namespace=local-cluster name=local-cluster\ntime=2026-03-04T10:36:58.877Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=137.303197ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/agent.open-cluster-management.io/v1/namespaces/local-cluster/klusterletaddonconfigs/local-cluster\ntime=2026-03-04T10:36:59.077Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=141.281084ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/agent.open-cluster-management.io/v1/namespaces/local-cluster/klusterletaddonconfigs/local-cluster\ntime=2026-03-04T10:36:59.300Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"config.open-cluster-management.io/v1alpha1, Kind=KlusterletConfig\" namespace=\"\" name=global\ntime=2026-03-04T10:36:59.477Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=118.265276ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/config.open-cluster-management.io/v1alpha1/klusterletconfigs/global\ntime=2026-03-04T10:36:59.692Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"cluster.open-cluster-management.io/v1beta2, Kind=ManagedClusterSet\" namespace=\"\" name=hypershift-management-clusters\ntime=2026-03-04T10:36:59.877Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=125.252738ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/cluster.open-cluster-management.io/v1beta2/managedclustersets/hypershift-management-clusters\ntime=2026-03-04T10:37:00.134Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-04T10:37:00.134Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:37:00.250Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config numVersions=1\ntime=2026-03-04T10:42:07.958Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T10:42:07.958Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:42:07.958Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:42:12.269Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config release=mce-config namespace=multicluster-engine status=failed description=\"Upgrade \\\"mce-config\\\" failed: post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-04T10:42:12.271Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yVwW/aMBTG7/krrJxASqqUVpuUqQfW7VR1k9adOu3g2A/j4rwX2QaExB8/mSQkhbFQ1lv1+X2/fs/P5EUGPLPgaGkFOHbHJPfc88LAiIoXEP5Bo8ydtxpVwmrpGy9hL8EKcKe4iotWHv+KGGMsfjBL58Ea8PeEM63ihMXKUMFN+CtO6qonsCstYCoELdGHk8Xel3IpCVNB6C0ZAzZdTUJFuTRei7osBVQa4TROWZFWZLTYNDjHL2IM2n6Qgc8apUbVeo7+tyUDRVfzdtr5gEeOXIG8rw+fYNfIfFOBdXM982m5Oy8BfdoAXH8ujS/ECDJVgG1dz5oPD+uNQGVF3si2Kb2AUF96k2iQ1rvnd+3zDO5Bu73RXs470fxf2O1VnnyrZzzSw9c5YPkClaFNyPrfv/XXqKP8HepSTmWp4op7GiJMpfyOHab73tVJ5uQ8yBCokf+Basb9uB/pjh1Mtb3N9rq7YbeiFVjkKKAlzCwvYU120Qd0H+1piN51Ykhw0z6nYyH6/SlaLAuwCB7c17AXXLRl6zlYYF6X4DwvK1aAXwMgG0nuIcijeJJNPqTZTZrd/rzO8puPeZY9x2N2dcVO1dxO8uvJczweR1v2QhrZQqO804hge9uMkHVLrL+9DtdWtGWVpXDaBU1YRbIuPg+SMEMqYSU4x1VAkpVgWbHpNS/BiT8BAAD//3mvoTFzBwAA\"\ntime=2026-03-04T10:42:12.273Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:42:12.274Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config step=deploy-mce-config error=\"error running Helm release deployment Step, failed to deploy helm release: post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T10:43:12.276Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-04T10:43:13.034Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-04T10:43:13.034Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:43:13.324Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-04T10:43:13.550Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:43:13.550Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:43:13.721Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config numVersions=2\ntime=2026-03-04T10:43:14.612Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:43:14.612Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"/v1, Kind=ServiceAccount\" namespace=multicluster-engine name=klusterlet-addon-controller-v2\ntime=2026-03-04T10:43:14.893Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"/v1, Kind=ServiceAccount\" namespace=multicluster-engine name=grc-policy-addon-sa\ntime=2026-03-04T10:43:15.059Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"/v1, Kind=ServiceAccount\" namespace=multicluster-engine name=grc-sa\ntime=2026-03-04T10:43:15.227Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=open-cluster-management:klusterlet-addon-controller-v2\ntime=2026-03-04T10:43:15.724Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=open-cluster-management:grc:clusterrole\ntime=2026-03-04T10:43:15.851Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=open-cluster-management:grc:policyaddon-clusterrole\ntime=2026-03-04T10:43:15.974Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=open-cluster-management:klusterlet-addon-controller-v2\ntime=2026-03-04T10:43:16.275Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=multicluster-engine name=open-cluster-management:grc:clusterrolebinding\ntime=2026-03-04T10:43:16.508Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=open-cluster-management:grc:policyaddon-clusterrolebinding\ntime=2026-03-04T10:43:16.626Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=multicluster-engine name=grc-grc-policy-addon-role\ntime=2026-03-04T10:43:17.166Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=multicluster-engine name=grc-grc-role\ntime=2026-03-04T10:43:17.289Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=multicluster-engine name=grc-grc-policy-addon-rolebinding\ntime=2026-03-04T10:43:17.605Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=multicluster-engine name=grc-grc-rolebinding\ntime=2026-03-04T10:43:17.723Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"apps/v1, Kind=Deployment\" namespace=multicluster-engine name=klusterlet-addon-controller-v2\ntime=2026-03-04T10:43:17.815Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=91.255291ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/klusterlet-addon-controller-v2\ntime=2026-03-04T10:43:18.015Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=139.301768ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/klusterlet-addon-controller-v2\ntime=2026-03-04T10:43:18.228Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"apps/v1, Kind=Deployment\" namespace=multicluster-engine name=grc-policy-addon-controller\ntime=2026-03-04T10:43:18.414Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=125.284813ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/grc-policy-addon-controller\ntime=2026-03-04T10:43:18.478Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"apps/v1, Kind=Deployment\" namespace=multicluster-engine name=grc-policy-propagator\ntime=2026-03-04T10:43:18.614Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=136.27702ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/grc-policy-propagator\ntime=2026-03-04T10:43:18.815Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=140.259156ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/apps/v1/namespaces/multicluster-engine/deployments/grc-policy-propagator\ntime=2026-03-04T10:43:18.878Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"addon.open-cluster-management.io/v1alpha1, Kind=AddOnDeploymentConfig\" namespace=multicluster-engine name=addon-hosted-config\ntime=2026-03-04T10:43:19.015Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=136.303591ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/namespaces/multicluster-engine/addondeploymentconfigs/addon-hosted-config\ntime=2026-03-04T10:43:19.214Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=138.355599ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/namespaces/multicluster-engine/addondeploymentconfigs/addon-hosted-config\ntime=2026-03-04T10:43:19.526Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"addon.open-cluster-management.io/v1alpha1, Kind=ClusterManagementAddOn\" namespace=\"\" name=config-policy-controller\ntime=2026-03-04T10:43:19.849Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"addon.open-cluster-management.io/v1alpha1, Kind=ClusterManagementAddOn\" namespace=\"\" name=governance-policy-framework\ntime=2026-03-04T10:43:20.014Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=105.268597ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/addon.open-cluster-management.io/v1alpha1/clustermanagementaddons/governance-policy-framework\ntime=2026-03-04T10:43:20.075Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"agent.open-cluster-management.io/v1, Kind=KlusterletAddonConfig\" namespace=local-cluster name=local-cluster\ntime=2026-03-04T10:43:20.215Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=139.295867ms reason=\"client-side throttling, not priority and fairness\" verb=GET URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/agent.open-cluster-management.io/v1/namespaces/local-cluster/klusterletaddonconfigs/local-cluster\ntime=2026-03-04T10:43:20.414Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=138.236517ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/agent.open-cluster-management.io/v1/namespaces/local-cluster/klusterletaddonconfigs/local-cluster\ntime=2026-03-04T10:43:20.657Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"config.open-cluster-management.io/v1alpha1, Kind=KlusterletConfig\" namespace=\"\" name=global\ntime=2026-03-04T10:43:20.814Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=97.280774ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/config.open-cluster-management.io/v1alpha1/klusterletconfigs/global\ntime=2026-03-04T10:43:21.034Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config gvk=\"cluster.open-cluster-management.io/v1beta2, Kind=ManagedClusterSet\" namespace=\"\" name=hypershift-management-clusters\ntime=2026-03-04T10:43:21.214Z level=DEBUG+1 msg=\"Waited before sending request\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config delay=120.320229ms reason=\"client-side throttling, not priority and fairness\" verb=PUT URL=https://prow-j5932672-mgmt-1-ykb7rkd7.hcp.westus3.azmk8s.io:443/apis/cluster.open-cluster-management.io/v1beta2/managedclustersets/hypershift-management-clusters\ntime=2026-03-04T10:43:21.510Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-04T10:43:21.510Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:43:21.677Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config numVersions=2\ntime=2026-03-04T10:48:29.332Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T10:48:29.332Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:48:29.332Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:48:34.292Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config release=mce-config namespace=multicluster-engine status=failed description=\"Upgrade \\\"mce-config\\\" failed: post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-04T10:48:34.294Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yVzW7bOhCF93qKgVYxIAWOE1xcqMgiTbsK2gJNVym6oMgJzZiaEUjahgE/fEFLslT/1K6RhTfHPJ/OcDiYxGIAh57nTqKHe1AiiCBKi1dcvqEMT4ZU4YMzpDNopK+iwq2EC6SN4mshO3n0MwEASB/t3Ad039limkHKNVIuGy2vBAmNFVIoZo1kMeRCKaZcMgXH1qLLF5PoTLN/Amoni1Z27dELCDVbI1dtosO0DqOdzOOvsbRVdIeruQ2m+waSNoTH/GdYPmFteRVDxoOnr+5s1F7+HnUpp3ZcCy0CnyI8KPWNeswj06vR0dMkmbIPqGKgVv4Lqm3wl20vN+xoauxdtj+rO+3WvEBHgiR2hFcnKlyymw0BT9uGPMTofSWWpbDdOzog7D3Pj4aUIf3ug3MGd2d+yt5xOe/INB1g70CPztbAemLEDtDOB/Qd7ZupLZfCDkM3L0a19/KMm1GYrmp0fmpew+A2uur90P6MbmEkPkjJ83eY7H3c3g16cRHjuC359SGZzUt0hAH957gXfLKG5RQdQjAV+iCqGkoMS0SCKyUCRvkqnYwn/+Xj23x89+NmXNzdFpObl3QE19dw9Mz/xe3dSzoaJWt4Y0MwM6TuDRG6wTZjgn6JDbfX7tpK1lA7jv/2QTOoWTWHz4NkYFlnUKH3QkckO4UOytWgeIVe/g4AAP//yqEAiXMHAAA=\"\ntime=2026-03-04T10:48:34.295Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-04T10:48:34.295Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config step=deploy-mce-config error=\"error running Helm release deployment Step, failed to deploy helm release: post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-04T10:49:34.303Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a5047a1b8f239fcf3946ccb9b23297b300325bca7da52e23ff104f18344d4840","failure_pattern":"post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: post-upgrade hooks failed: resource not ready, name: finalize-mce-config, kind: Job, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a5047a1b8f239fcf3946ccb9b23297b300325bca7da52e23ff104f18344d4840"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4312/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029124303575846912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4312,"pr_state":"closed","pr_sha":"9026867d4467f13a9d8bf4cbb45024d4bf8d1562","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T09:18:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"487a1d5931e88f4001d349ddcc251d33f70642246b20d53a18ccd3cb489241a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4312/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029124303575846912","occurred_at":"2026-03-04T09:18:27Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T09:42:31.190Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T09:42:33.466Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T09:42:34.627Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b8cb79f8451e822112431e08375e086920b2c10cb6ed0a737b16df920f754435\ntime=2026-03-04T09:44:05.376Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5846912/providers/Microsoft.Resources/deployments/b8cb79f8451e822112431e08375e086920b2c10cb6ed0a737b16df920f754435/operationStatuses/08584289895315685810\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5846912/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5846912/providers/Microsoft.Monitor/accounts/hcps-j5846912/providers/Microsoft.Authorization/roleAssignments/ed28c9fa-e3ec-515e-83d1-fbc466c6a362\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T094247Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '50d2b453-bde4-4089-8d1b-88157a91f991', request correlation id 'f013c198-3840-4975-bd48-c454b9d90887'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4304/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029119569045491712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4304,"pr_state":"closed","pr_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","final_merged_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T08:59:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029118848355012608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4304,"pr_state":"closed","pr_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","final_merged_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T08:56:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029118121138196480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T08:53:52Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9a5a7221aca672e3f551a91179b168b5043c987f3955b93b8a56ba306c436309","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029118121138196480","occurred_at":"2026-03-04T08:53:52Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T09:25:10.209Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T09:25:12.906Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T09:25:14.221Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c9c13d8d3f177007ffa536c5a0c0178407f8c3dd5273a8d806544a5349af29e9\ntime=2026-03-04T09:26:45.099Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8196480/providers/Microsoft.Resources/deployments/c9c13d8d3f177007ffa536c5a0c0178407f8c3dd5273a8d806544a5349af29e9/operationStatuses/08584289905720284803\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8196480/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8196480/providers/Microsoft.Monitor/accounts/services-j8196480/providers/Microsoft.Authorization/roleAssignments/320965c8-f75a-575e-a9d7-0d05650a39e9\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T092535Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '060439e0-150f-427c-a38c-a333f76e592c', request correlation id '1236440e-4047-46d3-ab9d-a34c63bf4c6a'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8196480/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8196480/providers/Microsoft.Monitor/accounts/hcps-j8196480/providers/Microsoft.Authorization/roleAssignments/d9a62d63-ca67-5e4e-aac9-c9f7aaa090b9\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T092534Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'd57cfc36-963c-4c2f-afc2-f0fb92926b36', request correlation id '1236440e-4047-46d3-ab9d-a34c63bf4c6a'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029116508784824320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T08:47:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029116483501559808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T08:47:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029073517814747136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T05:56:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029072799162699776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-04T05:53:48Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"05dcaffbd3fb01afcc97054cc27bef520dabd34115630d1210d7cd05074f4d16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029072799162699776","occurred_at":"2026-03-04T05:53:48Z","failed_at":"e2e","signature_id":"006ca039326c10659c098bfca6e790ed765ddc186a8cbd045c5391830e8f0e50","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"oller.go:284\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f7c2de57260, 0xc00064ec00})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc00064ec00?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 89\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 08:18:21.788\n \"ts\"=\"2026-03-04 08:18:21.788612\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 08:18:21.788697\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-t9b7gc\" \"resourceGroup\"=\"customer-rg-86vcln\"\n \"ts\"=\"2026-03-04 08:18:22.518338\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-t9b7gc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 08:18:23.236416\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-h765xv\" \"resourceGroup\"=\"customer-rg-86vcln\"\n \"ts\"=\"2026-03-04 08:18:23.392750\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-h765xv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 08:18:24.093594\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 08:18:24.093734\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 08:18:24.093823\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 08:18:24.095674\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-72\"\n \"ts\"=\"2026-03-04 08:18:24.095703\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-21\"\n \"ts\"=\"2026-03-04 08:18:24.095721\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-86vcln\"\n \"ts\"=\"2026-03-04 08:18:24.095694\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-56\"\n \"ts\"=\"2026-03-04 08:18:24.744965\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-72\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 08:18:25.228603\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 08:18:25.539061\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-21\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 08:18:25.977987\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-56\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 08:18:26.430427\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-72\" \"deployment\"=\"msi-deployment-72\"\n \"ts\"=\"2026-03-04 08:18:26.742963\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-h765xv\"\n \"ts\"=\"2026-03-04 08:18:27.218185\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-21\" \"deployment\"=\"msi-deployment-21\"\n \"ts\"=\"2026-03-04 08:18:27.640141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-56\" \"deployment\"=\"msi-deployment-56\"\n \"ts\"=\"2026-03-04 08:18:28.418117\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 08:18:29.600189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"customer-infra-basic-hcp-cluster-t9b7gc\"\n \"ts\"=\"2026-03-04 08:18:30.278630\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 08:18:30.278790\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 08:18:30.278935\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-86vcln\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\noller.go:284\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f7c2de57260, 0xc00064ec00})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc00064ec00?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 89\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 08:18:21.788\n \"ts\"=\"2026-03-04 08:18:21.788612\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 08:18:21.788697\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-t9b7gc\" \"resourceGroup\"=\"customer-rg-86vcln\"\n \"ts\"=\"2026-03-04 08:18:22.518338\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-t9b7gc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 08:18:23.236416\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-h765xv\" \"resourceGroup\"=\"customer-rg-86vcln\"\n \"ts\"=\"2026-03-04 08:18:23.392750\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-h765xv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 08:18:24.093594\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 08:18:24.093734\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 08:18:24.093823\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 08:18:24.095674\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-72\"\n \"ts\"=\"2026-03-04 08:18:24.095703\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-21\"\n \"ts\"=\"2026-03-04 08:18:24.095721\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-86vcln\"\n \"ts\"=\"2026-03-04 08:18:24.095694\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-56\"\n \"ts\"=\"2026-03-04 08:18:24.744965\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-72\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 08:18:25.228603\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 08:18:25.539061\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-21\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 08:18:25.977987\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-56\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 08:18:26.430427\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-72\" \"deployment\"=\"msi-deployment-72\"\n \"ts\"=\"2026-03-04 08:18:26.742963\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-h765xv\"\n \"ts\"=\"2026-03-04 08:18:27.218185\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-21\" \"deployment\"=\"msi-deployment-21\"\n \"ts\"=\"2026-03-04 08:18:27.640141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-56\" \"deployment\"=\"msi-deployment-56\"\n \"ts\"=\"2026-03-04 08:18:28.418117\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 08:18:29.600189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-86vcln\" \"deployment\"=\"customer-infra-basic-hcp-cluster-t9b7gc\"\n \"ts\"=\"2026-03-04 08:18:30.278630\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 08:18:30.278790\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 08:18:30.278935\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-86vcln\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029060435109482496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T05:04:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"61c3c5b14a0ce5d8902e23b36461bc0092b3b89a39a254821a50aaa2b0f109b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029060435109482496","occurred_at":"2026-03-04T05:04:40Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T05:36:43.059Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T05:36:45.713Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T05:36:46.792Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f4ed8eb32badc7234c2155c9ffa861442377110d421fd08fdfad2617b9a92a43\ntime=2026-03-04T05:38:47.732Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9482496/providers/Microsoft.Resources/deployments/f4ed8eb32badc7234c2155c9ffa861442377110d421fd08fdfad2617b9a92a43/operationStatuses/08584290042793385860\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9482496/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9482496/providers/Microsoft.Monitor/accounts/services-j9482496/providers/Microsoft.Authorization/roleAssignments/e7b2660f-1cf9-5e64-9dd3-68818bc1c9ac\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T053708Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a83d79bb-5c51-4dfa-939b-d0128782244e', request correlation id '5791217c-c4d5-4a2c-8730-912db8c89cd9'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029059733515669504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T05:01:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"82d7e02925736b438edbe916bc459492c037be4fd48126a18315b6f18834a458","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029059733515669504","occurred_at":"2026-03-04T05:01:53Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T05:36:41.406Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T05:36:44.342Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T05:36:45.509Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=163514b01592d3ff666266d7127ae3698b6d1f90d603e58fb1d301b0e6c30523\ntime=2026-03-04T05:38:16.486Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5669504/providers/Microsoft.Resources/deployments/163514b01592d3ff666266d7127ae3698b6d1f90d603e58fb1d301b0e6c30523/operationStatuses/08584290042806728132\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5669504/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5669504/providers/Microsoft.Monitor/accounts/hcps-j5669504/providers/Microsoft.Authorization/roleAssignments/58c9e49f-6d6b-5f1a-953e-551b7bd6d4af\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T053700Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'b67f1b3a-6678-446b-a192-5358792debb0', request correlation id '4612beac-82e5-41b3-ad78-1e5a9ed71d19'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029051669701988352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4309,"pr_state":"closed","pr_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","final_merged_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T04:29:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029023961471258624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T02:39:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029018189836324864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4266,"pr_state":"closed","pr_sha":"b7c66ef63434e9282b921ed6b480eca0fa792d64","final_merged_sha":"7fbd7a365361285c704975d46eb93dffa3ec17d6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T02:16:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2029017757130952704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4309,"pr_state":"closed","pr_sha":"080f4dd9a6214af07c5c6ded3a2bde9fc64a7f50","final_merged_sha":"b8a2fa00d1dfb356eb80a7d701af960b4f2589b2","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T02:15:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4304/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028999730922524672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4304,"pr_state":"closed","pr_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","final_merged_sha":"7dc387193a8168527a6670895d3fcf1d14b01184","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T01:03:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028999081933672448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4289,"pr_state":"closed","pr_sha":"2acf54cb98f656fdfbef25cb79ebd5af46e0b0f5","final_merged_sha":"f879b10dc600aa4bc317f4550e395d2e07047a53","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T01:00:36Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"59fc89da4d3e66dd0288cdf8a35ee8331da739d6ee5dc482757dc3aa4e167e28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028999081933672448","occurred_at":"2026-03-04T01:00:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028998788441444352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"1710a252e4122ac0d277a20159eadd6f0506149b","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T00:59:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6ea70ad23fe2663d7a5216c5da01971db1ed402f88eb756b1752940efcbd4673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028998788441444352","occurred_at":"2026-03-04T00:59:42Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-04T01:23:08.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-04T01:23:11.469Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-04T01:23:12.605Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ab9828817a590fcc565319d3b9f759ac33a2a189e9aee5854050ce875e1c7bb4\ntime=2026-03-04T01:25:13.543Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1444352/providers/Microsoft.Resources/deployments/ab9828817a590fcc565319d3b9f759ac33a2a189e9aee5854050ce875e1c7bb4/operationStatuses/08584290194935820161\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1444352/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1444352/providers/Microsoft.Monitor/accounts/hcps-j1444352/providers/Microsoft.Authorization/roleAssignments/c03f7507-bf88-5682-8e3c-44c67e4e4abd\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260304T012327Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'aca8e687-172e-44c1-b3a0-017cb0fe496c', request correlation id 'e5fbf330-df22-4396-b7bd-412e0e32d8a4'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028992733670019072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T00:35:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028992441536745472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T00:34:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028992090964234240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-04T00:33:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4297/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028991893244743680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4297,"pr_state":"closed","pr_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","final_merged_sha":"6420533dae683a630ecbf5a12aab2bbd9c355c8e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T00:32:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028989788740128768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"4825dcfa9ff220ccf8ea0ece43ff9cfe34ba2ace","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-04T00:23:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ad9890e1b905d91c085a57b76c2d96afefae574561f2396af6426a9fb9b5c10b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028989788740128768","occurred_at":"2026-03-04T00:23:54Z","failed_at":"provision","signature_id":"7acb328ca571196613fcc24303d5ec48dcd52f7ac5349cdf5ee6de30f5b52e6c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/kustoDataConnection","test_suite":"step graph","failure_text":"time=2026-03-04T01:03:13.616Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection description=\"Step kustoDataConnection\\n Kind: ARM\\n Template: templates/audit-logs-data-connection.bicep\\n Parameters: configurations/audit-logs-data-connection.tmpl.bicepparam\"\ntime=2026-03-04T01:03:14.606Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection\ntime=2026-03-04T01:03:15.530Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection deployment=a1c7026655439497a9842d043b8c6ee34a6002875ebdf9d55911262d5cbed8d2\ntime=2026-03-04T01:03:45.882Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/a1c7026655439497a9842d043b8c6ee34a6002875ebdf9d55911262d5cbed8d2/operationStatuses/08584290206903492434\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CanNotAlterEventHubResourceId\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0241280/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/westus3-shared-resources/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d75be1569a2db42bca9087fe5a7bd21fee0536b43d9658da65baebfa84fcd3dd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CanNotAlterEventHubResourceId; detail message [BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs; provider Microsoft.EventHub","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d75be1569a2db42bca9087fe5a7bd21fee0536b43d9658da65baebfa84fcd3dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4287/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028989510120902656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4287,"pr_state":"closed","pr_sha":"723b6e68710379451b6a670f5df9529985329888","final_merged_sha":"723b6e68710379451b6a670f5df9529985329888","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-04T00:22:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}