{"meta":{"date":"2026-03-02","timezone":"UTC","generated_at":"2026-11-01T00:09:10Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":64,"failed_runs":46,"runs_with_occurrences":46,"runs_with_failure_pattern_matches":45,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":18,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028618153788968960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4290,"pr_state":"closed","pr_sha":"e49ae07b456820c1978cffafbde806bdfea932d4","final_merged_sha":"507eb199f7a4d7137d6ae5eddc82d001167895fb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T23:47:12Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f76a72c5e614601b83604af3b059de26e62badd3f0bae23abed9cf640d84c353","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4290/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028618153788968960","occurred_at":"2026-03-02T23:47:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4285/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028615430284775424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4285,"pr_state":"closed","pr_sha":"0c04951d2c0e3a9e870c6002db3f38c1f8161eab","final_merged_sha":"b61beb928788a2673941120ce3c13c31afe9b930","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T23:36:23Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4b03f688a0a6e9c024a29df3385f3a41609c94866066892fd2fb9582d81c5ee4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4285/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028615430284775424","occurred_at":"2026-03-02T23:36:23Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T00:06:24.150Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T00:06:26.033Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T00:06:27.398Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=bb8b46d2344a06247650233535ed6b82223120355e7db17d1c2bc499ec124d2e\ntime=2026-03-03T00:07:58.262Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4775424/providers/Microsoft.Resources/deployments/bb8b46d2344a06247650233535ed6b82223120355e7db17d1c2bc499ec124d2e/operationStatuses/08584291104989036354\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4775424/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4775424/providers/Microsoft.Monitor/accounts/services-j4775424/providers/Microsoft.Authorization/roleAssignments/c00e6b00-62d0-5cf8-be1b-93cf135c790d\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T000659Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '8741617a-a047-490d-bdb9-8fbc601b7c7e', request correlation id 'a1f3e123-ec56-484c-8341-6e3d18029b31'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028613965700927488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4284,"pr_state":"closed","pr_sha":"a3cd5395168ba92adba577ff018ab176565cb0bb","final_merged_sha":"efd28c631288d18973d727e83e9a2b147a02894e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T23:30:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4932e6289ddd69bf726952b414ca003e047def344fae5ecf0f314e13c99bdbf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028613965700927488","occurred_at":"2026-03-02T23:30:33Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-03T00:03:19.295Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-03T00:03:21.361Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-03T00:03:22.592Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0144bc3d3ae2925c6c4f059d15818b3b6d719cc8e363559f0c191709eb665ce8\ntime=2026-03-03T00:05:23.532Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0927488/providers/Microsoft.Resources/deployments/0144bc3d3ae2925c6c4f059d15818b3b6d719cc8e363559f0c191709eb665ce8/operationStatuses/08584291106836397583\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0927488/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0927488/providers/Microsoft.Monitor/accounts/hcps-j0927488/providers/Microsoft.Authorization/roleAssignments/b3b1f314-a41d-547d-a0e6-0f981a81383b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260303T000343Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '89440e79-372a-4e05-92e6-402411eaedb7', request correlation id '099874e2-21ae-40b1-b0e3-72c1aa8cbbbd'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028613603325644800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4289,"pr_state":"closed","pr_sha":"2acf54cb98f656fdfbef25cb79ebd5af46e0b0f5","final_merged_sha":"f879b10dc600aa4bc317f4550e395d2e07047a53","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T23:29:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1923ec8ee60b6d2c83e2b43847c735b90d142f02f330dbe9e004f49eab317a2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4289/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028613603325644800","occurred_at":"2026-03-02T23:29:07Z","failed_at":"e2e","signature_id":"02ecb23823c2b6b5b5ee8d45a26fdbd47fdfe5de80a96d588e433aba9f2ad446","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000832680\u003e: \n failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n {\n msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n err: \u003c*errors.StatusError | 0xc000db8460\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n Reason: \"NotFound\",\n Details: {\n Name: \"aro-hcp-exporter\",\n Group: \"\",\n Kind: \"services\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 404,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000832680\u003e: \n failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n {\n msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n err: \u003c*errors.StatusError | 0xc000db8460\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n Reason: \"NotFound\",\n Details: {\n Name: \"aro-hcp-exporter\",\n Group: \"\",\n Kind: \"services\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 404,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028613190509662208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"37f2d56d806b70777b589cb699b3fe6c4aeabbd6","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T23:27:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bb39086f6475a6252accb8ab8baf42a80d2dd1444f555b768be7451ed0e0e0e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028613190509662208","occurred_at":"2026-03-02T23:27:29Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T23:58:26.596Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T23:58:28.307Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T23:58:29.546Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e3c00e6b78ab437cefc3c3b74d97bb948efd2f1adc86ebe6a8a0827e33b0508d\ntime=2026-03-03T00:00:30.549Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9662208/providers/Microsoft.Resources/deployments/e3c00e6b78ab437cefc3c3b74d97bb948efd2f1adc86ebe6a8a0827e33b0508d/operationStatuses/08584291109767219232\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9662208/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9662208/providers/Microsoft.Monitor/accounts/services-j9662208/providers/Microsoft.Authorization/roleAssignments/79e61e8c-4926-5639-8557-70ed12fa7754\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T235853Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9a03c66a-6ace-48d4-aad7-dceec3f4b86e', request correlation id 'd7f1bf5f-4dd0-4b61-896b-c8d7886cf853'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9662208/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9662208/providers/Microsoft.Monitor/accounts/hcps-j9662208/providers/Microsoft.Authorization/roleAssignments/faf589dc-9f78-5605-a683-9426e4dad6fc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T235852Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c632e93f-57d3-4fc7-99d0-9e5dcba7849c', request correlation id 'd7f1bf5f-4dd0-4b61-896b-c8d7886cf853'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028590871301066752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4062,"pr_state":"closed","pr_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","final_merged_sha":"26b2f214e2d6443a2d37ec7705c6a64a65050f60","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T21:58:47Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d730abfd35e5c94413e3ed2ff098dd1816e282fd97221778316addd425643dab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028590871301066752","occurred_at":"2026-03-02T21:58:47Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T22:53:50.831Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-02T22:55:26.141Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=e6d889cf-c88c-460b-ab27-d67d1ae156df \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=3a50d8ea-da62-412e-8dd5-06ab51680b08 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj1066752.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj1066752.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j1066752 -n arohcp-prow-dbcs-j1066752 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1066752/providers/Microsoft.Network/dnszones/j1066752.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j1066752-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j1066752-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set global.imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j1066752-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"c240f6f7-f7b1-4275-9b6a-a1d3b4d22f64\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j1066752-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1066752-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j1066752-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028581381784735744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4276,"pr_state":"closed","pr_sha":"a75bab5ee1d156ac57ff1f8b7d157b5e9f8a898c","final_merged_sha":"a75bab5ee1d156ac57ff1f8b7d157b5e9f8a898c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T21:21:05Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e58fbaccd340c92577e92b65a01a2478b9f278840a3f785d7bc60ac25e460e20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028581381784735744","occurred_at":"2026-03-02T21:21:05Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T21:57:04.680Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T21:57:07.057Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T21:57:08.203Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=b46cd716a2ca59e327c8a2967c2e034c634569976a37e9c616728f0eed670e0b\ntime=2026-03-02T21:58:39.136Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4735744/providers/Microsoft.Resources/deployments/b46cd716a2ca59e327c8a2967c2e034c634569976a37e9c616728f0eed670e0b/operationStatuses/08584291182579418242\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4735744/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4735744/providers/Microsoft.Monitor/accounts/hcps-j4735744/providers/Microsoft.Authorization/roleAssignments/1d2592c3-a20c-57cb-b4b3-3ed941a3c647\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T215727Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f91671b4-094c-4d12-b4f2-ebe8815c496d', request correlation id '90779488-248a-4756-824b-30e9191b8bca'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4282/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028577867083812864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4282,"pr_state":"closed","pr_sha":"678354eb35a5b9ddf268b2149c7e55b0dd617459","final_merged_sha":"678354eb35a5b9ddf268b2149c7e55b0dd617459","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T21:07:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3be6e5fb0a53c1be75b4608bacd894d75223b4583d2b2b9743ece4db309edff4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4282/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028577867083812864","occurred_at":"2026-03-02T21:07:07Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-02T21:54:44.252Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-02T21:54:45.068Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:157f9e45-d5b5-4321-9b93-3ff8c0d3578e]] prometheusSpec:map[externalLabels:map[cluster:prow-j3812864-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j3812864-mgmt-1-tgby.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-2e0ad25e914f439c8ec314454c3b59c7/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:375d727bf8137d574b5aad4f94881fade0a57556ddf52a3f98322e5d561286af] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j3812864-mgmt-1-tgby.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-b180572098fb409c878c41bf3e88e097/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-02T21:54:45.068Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T21:54:45.153Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/prometheus?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69a60504b61217000172150f.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69a60504b61217000172150f.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces prometheus)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36f11a42eb7aebd39734d0aa77ea84f9faa2e08835fd9e817514dec0e69cc819","failure_pattern":"failed to apply namespace prometheus: an error on the server (\"Internal Server Error: \\\\\"/api/v1/namespaces/prometheus?fieldManager=templatize\u0026amp;force=false\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443 while dialing","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace prometheus: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/prometheus?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69a60504b61217000172150f.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69a60504b61217000172150f.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces prometheus)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36f11a42eb7aebd39734d0aa77ea84f9faa2e08835fd9e817514dec0e69cc819"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4285/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028572207055114240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4285,"pr_state":"closed","pr_sha":"74ef9a9b397105afe9c8ecef9242203abbc8dd1f","final_merged_sha":"b61beb928788a2673941120ce3c13c31afe9b930","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T20:44:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fe5813496fc702741dd711a9b19b1d3add4570e372b88d4b7caf681fdb65be32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4285/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028572207055114240","occurred_at":"2026-03-02T20:44:27Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-02T21:38:07.196Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-02T21:38:07.686Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-02T21:38:07.686Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T21:38:07.747Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-02T21:38:07.864Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-02T21:38:07.864Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T21:38:07.978Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T21:40:08.986Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: Unauthorized\"\ntime=2026-03-02T21:40:08.986Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T21:40:08.986Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T21:40:14.826Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce release=mce namespace=multicluster-engine status=failed description=\"Release \\\"mce\\\" failed: failed post-install: Unauthorized\" values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-02T21:40:14.827Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yST2/TQBDF7/4UI58Sya4cx8EhqBcQJwQHyqmIw3p3Ym+9/zo7TuQqHx6ZUBpFuOLA9fmn3xuvXmKQgTD6gSRGuAUlWLBoDC5884CSP2mndpFJuzaDc/RFWPwT4QHdryQGIZ/j5fcEACC9Q0nIaQapHQxraYbISDm6VjvMfUAS7Ck/YtN5389waXaWfTinX73BOWMYOlut68Mm+LfV6KvggmrrzaYSuqzJPNnKbofNY1c3alL8xfxeO6Vd+58LXru5rMYw1P1+5DhSuaqc1r4jHIZ+W2Jv1/vjE3XeRHycUzy33CEdtJwtun7uPL7KX1k/e6fZ0wT/5ozeoxylwTyyYMwtMmkZ80M5Z0x+vEv6oUFyyBg/TsuJyQmOHRICa4uRhQ3QIB8RHSyUYJziRVoW5Zu8WOdF+a1c7dbbXVHfp0u4uYE5pip2q+o+XS6TEzx47aDXTt1q55Au9u4dvMz8ct/Xw05OEMhPX18OzSB4dYb/TZKB8W0GFmMU7aT0pJCgGS9+XmGUPwMAAP//WG5mbZUDAAA=\"\ntime=2026-03-02T21:40:14.827Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T21:40:14.827Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028569542367645696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T20:31:31Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"44b49c2953f73bccff827e98fc2620bad52d72a0f0bcdbd1446c449cb0c70e17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028569542367645696","occurred_at":"2026-03-02T20:31:31Z","failed_at":"e2e","signature_id":"f44fb1d7fbd9917bd0c2ceb0cb4ff2ae900bc93cc783108a6affd8fc270888c8","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e4060\u003e: \n failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n {\n msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n err: \u003c*errors.StatusError | 0xc000e47040\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n Reason: \"NotFound\",\n Details: {\n Name: \"aro-hcp-exporter\",\n Group: \"\",\n Kind: \"services\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 404,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e4060\u003e: \n failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n {\n msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n err: \u003c*errors.StatusError | 0xc000e47040\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n Reason: \"NotFound\",\n Details: {\n Name: \"aro-hcp-exporter\",\n Group: \"\",\n Kind: \"services\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 404,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}},{"row_id":"c7c259df31b7d773e873d449f95c5d4018babcf2c23c9206ed6d65dca456812a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028569542367645696","occurred_at":"2026-03-02T20:31:31Z","failed_at":"e2e","signature_id":"ab5789b3ea73370c611d9b2cac95fe8e536f1660f6792a157acee1adb25375d2","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.001s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00053a240\u003e: \n nfd-worker DaemonSet not found\n {\n s: \"nfd-worker DaemonSet not found\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.001s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00053a240\u003e: \n nfd-worker DaemonSet not found\n {\n s: \"nfd-worker DaemonSet not found\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb","failure_pattern":"nfd-worker DaemonSet not found","search_query":"nfd-worker DaemonSet not found"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028566508652728320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4284,"pr_state":"closed","pr_sha":"bfae6ae18b022bcb307d5cf6452d4c69e1d2d1ea","final_merged_sha":"efd28c631288d18973d727e83e9a2b147a02894e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T20:19:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e03e6f518e1627cdc120f6e1fcfbdce825d8650a102fc11bec9e112980d3d573","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4284/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028566508652728320","occurred_at":"2026-03-02T20:19:10Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T20:50:00.254Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T20:50:02.576Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T20:50:03.708Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cc69a4904ada8258f28c76e74d2e465a4796cdd3d0f86c7b804d868309638d29\ntime=2026-03-02T20:51:34.629Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2728320/providers/Microsoft.Resources/deployments/cc69a4904ada8258f28c76e74d2e465a4796cdd3d0f86c7b804d868309638d29/operationStatuses/08584291222824713303\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2728320/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2728320/providers/Microsoft.Monitor/accounts/hcps-j2728320/providers/Microsoft.Authorization/roleAssignments/58e6ac77-0806-5c61-bd9a-9a28a2ff2f52\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T205024Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '13350b18-acb3-4461-ab93-5ac5f7deb17a', request correlation id '16354f02-ed0f-4688-a296-9d3c5e6481fa'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028563615811899392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"ce89a8ba8a20780ef7cf82d351c9c431ff7fe244","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T20:00:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"26010b9de95891bd69ca29aacbe85ea896a71161cba767aeba2347cf9318a9ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028563615811899392","occurred_at":"2026-03-02T20:00:56Z","failed_at":"provision","signature_id":"267bf030a61dbcb7455ab84c6db6581482a98474287230199f58904612b41a7f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T20:28:40.574Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T20:28:41.279Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy values=\"map[azure:map[clientId:a30e70b9-0887-4141-a184-e5da66cc18ad cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-prow-maestro-j1899392.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j1899392-mgmt-1 credsKeyVault:map[name:ah-prow-mg-j1899392-1 secret:hcp-underlay-j1899392-mgmt-1] deployment:map[zoneCount:4] glog_v:2 image:map[digest:sha256:144f3abdc7e843bf8a59b3b45c9892528e22de6068567fb6cfd26630f33f621e registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:3 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:2dccfca276d1e7611442dc997ac15055f76c4caff92b668b40907cdd68fffa14 registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-03-02T20:28:41.279Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy\ntime=2026-03-02T20:28:41.769Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace maestro: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7478c3bf6f7789af26345a84bea8dcc29910ad45614b480a328559b0ed831564","failure_pattern":"failed to apply namespace maestro: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace maestro: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7478c3bf6f7789af26345a84bea8dcc29910ad45614b480a328559b0ed831564"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028560006214848512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4276,"pr_state":"closed","pr_sha":"a75bab5ee1d156ac57ff1f8b7d157b5e9f8a898c","final_merged_sha":"a75bab5ee1d156ac57ff1f8b7d157b5e9f8a898c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T19:46:31Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ad1235f9c654fa18dfec8bec2af4a868c6d777cb67b7888c6496924dc10cc206","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028560006214848512","occurred_at":"2026-03-02T19:46:31Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-02T20:42:33.406Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-02T20:42:34.085Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-02T20:42:34.085Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T20:42:34.147Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-02T20:42:34.262Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-02T20:42:34.262Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T20:42:34.375Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T20:46:16.002Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: Unauthorized\"\ntime=2026-03-02T20:46:16.002Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T20:46:16.002Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T20:46:16.629Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed to get status for release mce: query: failed to query with labels: Unauthorized\"\ntime=2026-03-02T20:46:16.629Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T20:46:16.630Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"137cd763836b5634bcfd021165c430a950c35b57adca01c06a2061d9b39a9971","failure_pattern":"failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: warning: Hook post-install multicluster-engine/templates/finalize-mce.job.yaml failed: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["137cd763836b5634bcfd021165c430a950c35b57adca01c06a2061d9b39a9971"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028564107313025024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4258,"pr_state":"closed","pr_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","final_merged_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T18:29:08Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"58773fa0d4edb566a5487022f544236d0fd0303a9a5248fbeb58fb1c4ef31716","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028564107313025024","occurred_at":"2026-03-02T18:29:08Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T20:38:12.599Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-02T20:38:30.352Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=94a25c73-da53-4013-a8b3-a655b8cbe3f1 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=7cf8bbed-e1fe-47a0-8028-5744e80cfbc6 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj3025024.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj3025024.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j3025024 -n arohcp-prow-dbcs-j3025024 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3025024/providers/Microsoft.Network/dnszones/j3025024.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j3025024-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j3025024-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set global.imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j3025024-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"449497a8-3fb2-4274-953d-ff054f7add6d\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j3025024-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3025024-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j3025024-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: Unable to continue with install: could not get information about the resource Deployment \\\"clusters-service\\\" in namespace \\\"clusters-service\\\": Unauthorized\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028541179544473600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4254,"pr_state":"closed","pr_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","final_merged_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T18:28:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d7feb13972da229261031d254fc9e960d466b84afe8a0017223f7c00ff73f434","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028541179544473600","occurred_at":"2026-03-02T18:28:12Z","failed_at":"provision","signature_id":"5564627504dc0ffa8b2a1cfb61f4b200a8883ded65fa2c1a4914a1dd7bd7f88c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce-config","test_suite":"step graph","failure_text":"time=2026-03-02T19:27:31.017Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config description=\"Step deploy-mce-config\\n Kind: Helm\\n\"\ntime=2026-03-02T19:27:31.721Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f global:map[registryOverride:arohcpsvcdev.azurecr.io/acm-d-cache] localCluster:map[addonConfig:map[applicationManagerEnabled:false certPolicyControllerEnabled:false policyControllerEnabled:true searchCollectorEnabled:false] kubeApiUrl:https://kubernetes.default.svc] policy:map[cluster-lifecycle:map[hubconfig:map[replicaCount:2]] grc:map[hubconfig:map[replicaCount:2]]]]\"\ntime=2026-03-02T19:27:31.721Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-02T19:27:31.830Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config namespace=multicluster-engine\ntime=2026-03-02T19:27:31.946Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config dryRun=false\ntime=2026-03-02T19:27:31.946Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-02T19:27:32.065Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-02T19:27:32.515Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"\ntime=2026-03-02T19:27:32.515Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-02T19:27:32.515Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-02T19:27:32.633Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"failed to get status for release mce-config: release: not found\"\ntime=2026-03-02T19:27:32.633Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config\ntime=2026-03-02T19:27:32.634Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce-config err=\"error running Helm release deployment Step, failed to deploy helm release: cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df1532ddcdc2d62f97589fa9c092641092a3064f3184196911681c8bfee0ae44","failure_pattern":"cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials","search_query":"error running Helm release deployment Step, failed to deploy helm release: cluster reachability check failed: kubernetes cluster unreachable: the server has asked for the client to provide credentials"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df1532ddcdc2d62f97589fa9c092641092a3064f3184196911681c8bfee0ae44"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028537841486663680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"cd42d690d61fe3044f9a6f43788fbc6776287e69","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T18:26:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ac68e218375397130e7cf77977ceeaa90f6d596979bfde233da49be457aef805","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028537841486663680","occurred_at":"2026-03-02T18:26:54Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T19:05:49.226Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T19:05:51.649Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T19:05:52.886Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=51710af12600844e0cd106dd8d061edf5a934e130b2c2cc662c1dc5c3fb700ce\ntime=2026-03-02T19:07:53.958Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6663680/providers/Microsoft.Resources/deployments/51710af12600844e0cd106dd8d061edf5a934e130b2c2cc662c1dc5c3fb700ce/operationStatuses/08584291285333124880\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6663680/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6663680/providers/Microsoft.Monitor/accounts/hcps-j6663680/providers/Microsoft.Authorization/roleAssignments/490d582e-2b21-5908-8365-6e9ad1f18118\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T190614Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c6945255-580a-471b-bfa3-9c4a6c38acb3', request correlation id '366b8aae-4e77-48fc-b17c-960eee8d024d'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6663680/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6663680/providers/Microsoft.Monitor/accounts/services-j6663680/providers/Microsoft.Authorization/roleAssignments/1e37ae5d-b94b-565c-a881-38ad69bbb497\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T190605Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c061ce31-071e-4ff0-be58-7022e4f0c1a6', request correlation id '366b8aae-4e77-48fc-b17c-960eee8d024d'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028541037001052160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"ce89a8ba8a20780ef7cf82d351c9c431ff7fe244","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T18:19:23Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"30cb6db7b0a6b90d9e75af33ab935bf7605eb5562f47f077db938d2bbb211030","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028541037001052160","occurred_at":"2026-03-02T18:19:23Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T19:22:11.796Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T19:22:14.441Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T19:22:15.704Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7ea01bcf5c148b04295f6841d8c2870fc8cd1ec283d684432c83b430e3a234a1\ntime=2026-03-02T19:23:46.511Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1052160/providers/Microsoft.Resources/deployments/7ea01bcf5c148b04295f6841d8c2870fc8cd1ec283d684432c83b430e3a234a1/operationStatuses/08584291275505391573\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1052160/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1052160/providers/Microsoft.Monitor/accounts/services-j1052160/providers/Microsoft.Authorization/roleAssignments/ffe2b569-d5f4-598e-b90f-b053dfa05f72\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T192231Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '2df28e68-ce51-4174-89b5-87c4a2445df5', request correlation id '5fcdac47-c27a-4970-b43b-2b08494f1d9f'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028544461314723840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"open","pr_sha":"c853e9af1512d4f8471ce2fa95b25b1ce6f48985","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T18:15:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"53d0db1530480006265413290557878bc7ec361036cbcf86338a297d37bcda75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028544461314723840","occurred_at":"2026-03-02T18:15:34Z","failed_at":"provision","signature_id":"a5b5f71a061e60f11c9c511d431eae042d41346983d1442cca50d855704dabbe","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/prometheus","test_suite":"step graph","failure_text":"time=2026-03-02T19:37:02.501Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-03-02T19:37:03.186Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:54511b11-7726-4664-8c71-7511e266fabd]] prometheusSpec:map[externalLabels:map[cluster:prow-j4723840-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j4723840-mgmt-1-hol8.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-09f30f5e35c1470496b3bf72b1fc8022/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:375d727bf8137d574b5aad4f94881fade0a57556ddf52a3f98322e5d561286af] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j4723840-mgmt-1-hol8.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-ddfae9ae51cc4a688d854461ebead646/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-02T19:37:03.187Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T19:37:03.530Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus namespace=prometheus\ntime=2026-03-02T19:37:03.870Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus dryRun=false\ntime=2026-03-02T19:37:03.870Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T19:37:04.036Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T19:38:15.656Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"failed post-install: Unauthorized\"\ntime=2026-03-02T19:38:15.656Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T19:38:15.656Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T19:38:24.268Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus release=arohcp-monitor namespace=prometheus status=failed description=\"Release \\\"arohcp-monitor\\\" failed: failed post-install: Unauthorized\" values=\"map[environment:arohcpprow kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[customResourceState:map[config:map[spec:map[resources:[map[groupVersionKind:map[group:hypershift.openshift.io kind:HostedCluster version:v1beta1] labelsFromPath:map[_id:[spec clusterID] api_url:[spec services [service=APIServer] servicePublishingStrategy route hostname] name:[metadata name] namespace:[metadata namespace]] metricNamePrefix:hostedClusterAPI metrics:[map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=KubeAPIServerAvailable] status]] type:StateSet] help:KubeAPIServer availability condition name:kubeapiserver_available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Available] status]] type:StateSet] help:Cluster availability condition name:available] map[each:map[stateSet:map[labelName:status list:[True False Unknown] path:[status conditions [type=Progressing] status]] type:StateSet] help:Cluster progressing condition name:progressing]]]]]] create:true enabled:true key:config.yaml name:kube-state-metric-custom-resource-config] image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:17fc37d9fe70a0efa4fdaf609bd03299013faa0fee16b77815469eb37a879e7c] rbac:map[extraRules:[map[apiGroups:[hypershift.openshift.io] resources:[hostedclusters] verbs:[list watch]]]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:7c74a715af2c94cb734785b4d3ea1357b4f02b88e1e123c622a9cb68b62f669c]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:b53be343389b63fa880639565d6067ba933f3f009642cd2e2c02bcdc189a63de] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:9f029892d072c33a88f7120170c900a3f2e6b9758f16477c5dec60c3ae7504e6]] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[prometheusSpec:map[replicas:2 shards:1] serviceAccount:map[managedIdentity:54511b11-7726-4664-8c71-7511e266fabd]] prometheusSpec:map[externalLabels:map[cluster:prow-j4723840-mgmt-1] hcpRemoteWriteUrl:https://msprom-westus3-prow-j4723840-mgmt-1-hol8.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-09f30f5e35c1470496b3bf72b1fc8022/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:375d727bf8137d574b5aad4f94881fade0a57556ddf52a3f98322e5d561286af] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-prow-j4723840-mgmt-1-hol8.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-ddfae9ae51cc4a688d854461ebead646/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 resources:map[limits:map[cpu:NONE memory:NONE] requests:map[cpu:NONE memory:NONE]] retention:1d retentionSize:45GiB version: zoneCount:4]]\"\ntime=2026-03-02T19:38:24.271Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6yWzW7bOBDH73oKwqcYMA2vs9jNepFDv05FiiAtWiBFD2NyKjMWOcRwFNeAH76QldiKXSWGlKNGf/7E+dJMVqAoxkQlG0zqUlkQEJgXeEbzOzTy0QU7S8Iu5CNVmz6Bx50J7zFsLSmCeTQPv2dKKTX4CoWzIC7k33C+IFq+o/DT5SWDOAqDkRpEJo+ywDJpsN6l9GAfjGrC9e79U/XhU63+jHzvDF5RcEJ8wKeIDEfmg6NvjKEySKUBpoWJ2tcwvSznqJOAoPYo7Mzzd2iAetzhJYfreF5B3F7Yw+PVdEKpwp602Sp8rah9WCdBvyMUZRLkGyrwZKdbzrb4+aK6KXoIQY/4v8dY0NpjryQ+hZycwGuyf6y9FvXOclMeRRAKZPEQIEcec1lgF0idfM1YEFjkLojlRao/PzYUBFxA1iaWukyQo05oKNikhQSKV6J79MRrbcAsjgqlH5LTa0XgAZhWdV+9HnFFvHQhr/pXz9fSKet7fCSraRWwrVyfpVSt0njOMSBD0bkWD3mBLGpGQ2xdyPthT+joFyEcUDBpiLEnIQkx5J1KtwHZ/qQ1RJeQ7zsn8CmtshR4NFi6sDowtgnHX5FYevzStpSAUjVKx+Odv336HDiN0jLT93PyrQtVc/QZzA1E23zOfvyf7RP8oVroUrZRqwUyKnEek4CPao6yQgzqzIJgZT4bTCfTf/TkXE+mX/76b3b+72xyfjsYqvFYtWouZtO/bwfDYbZRd+SCWrpgL10IyI01lILab5/NtfNw38w2KjJVb/cXHalIthafBhmpgvKR8piqmZZtFLFFVvN1w3mLyfwOAAD//x7gcNosCwAA\"\ntime=2026-03-02T19:38:24.273Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus\ntime=2026-03-02T19:38:24.274Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028532529341927424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T18:06:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c91e6151e27481c56061fb854d811189c95b708ffd66a95bfb79ef1f1e40abb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028532529341927424","occurred_at":"2026-03-02T18:06:27Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T18:42:10.894Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T18:42:13.167Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T18:42:14.712Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cad5f7ea5978e9a723a7ef9f7e590d49aadff59bac89cdd7a088446645d724a0\ntime=2026-03-02T18:44:15.923Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1927424/providers/Microsoft.Resources/deployments/cad5f7ea5978e9a723a7ef9f7e590d49aadff59bac89cdd7a088446645d724a0/operationStatuses/08584291299517547504\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1927424/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1927424/providers/Microsoft.Monitor/accounts/services-j1927424/providers/Microsoft.Authorization/roleAssignments/4302db2f-dbd2-5136-abfc-6b017bbc2603\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T184236Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '253b5600-9f48-4b72-bd20-0cf6bef3fb9d', request correlation id '22add12f-3ac8-41b2-8a9e-76897f9aa58e'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028534693640540160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T17:46:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2c68b36115cb92d8d07c9e966acbb2de37f00be8a9c08a640c90d37bd7dbc4df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028534693640540160","occurred_at":"2026-03-02T17:46:18Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T18:26:39.004Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T18:26:41.640Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T18:26:43.132Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9c55382e68e6f16d3a9cd46566c62e9c6bf932308e2e2971284f6f69b2bffbcc\ntime=2026-03-02T18:28:13.879Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0540160/providers/Microsoft.Resources/deployments/9c55382e68e6f16d3a9cd46566c62e9c6bf932308e2e2971284f6f69b2bffbcc/operationStatuses/08584291308830961753\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0540160/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0540160/providers/Microsoft.Monitor/accounts/services-j0540160/providers/Microsoft.Authorization/roleAssignments/1bdd06c2-e90e-5a90-b32e-5fa4eab0c935\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T182707Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '3f9e2b4c-9567-45c5-bd94-b4b96db17e61', request correlation id 'd1e84e8d-4e77-45ac-9e8b-b883058e1ff0'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028515355458015232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4279,"pr_state":"closed","pr_sha":"544b8a8d6204dcdade54eae218447f69fea67935","final_merged_sha":"efa8e746c645a87051488818bbf21f67fb2329fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T16:58:43Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"adf514df18bb34f65879450c694cb536f92daf0076e4966a825490ff9f117ceb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028515355458015232","occurred_at":"2026-03-02T16:58:43Z","failed_at":"provision","signature_id":"692850ccec3627f6f18c0069e48528a9b9f4e27dd84dcc0a7a087aa7c2d7afa7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-mce","test_suite":"step graph","failure_text":"time=2026-03-02T17:54:42.466Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce description=\"Step deploy-mce\\n Kind: Helm\\n\"\ntime=2026-03-02T17:54:43.072Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce values=\"map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:1c49cb26c44ce9f846257ca1bae2d275862ebeb31d5529a62a6c4075665fd38f imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]\"\ntime=2026-03-02T17:54:43.072Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T17:54:43.131Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce namespace=multicluster-engine\ntime=2026-03-02T17:54:43.247Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce dryRun=false\ntime=2026-03-02T17:54:43.247Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T17:54:43.360Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T17:56:44.546Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed post-install: Unauthorized\"\ntime=2026-03-02T17:56:44.546Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T17:56:44.546Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T17:56:45.145Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"failed to get status for release mce: query: failed to query with labels: Unauthorized\"\ntime=2026-03-02T17:56:45.145Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce\ntime=2026-03-02T17:56:45.145Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6","failure_pattern":"failed post-install: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82e6580bc32e56aa7378422fd2960d02a2eebbb9fbfbc08994384c40137bbab6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028510678175715328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4181,"pr_state":"closed","pr_sha":"e8fec6bb18b616d66d477e112301f58adcdc2bcd","final_merged_sha":"0ac5e198ed7921a253ae14ad2c7ea366e71f81ce","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T16:40:08Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"89ed3d67916648794217cf2efd143ecd5a61ecac64db8f8568d29a86e0914519","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028510678175715328","occurred_at":"2026-03-02T16:40:08Z","failed_at":"provision","signature_id":"b04de30e7ba1a68269721053e97cd6341141d39fce43570f2847848e9345b67c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ACM/management/deploy-policies","test_suite":"step graph","failure_text":"time=2026-03-02T17:36:35.063Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies description=\"Step deploy-policies\\n Kind: Helm\\n\"\ntime=2026-03-02T17:36:35.752Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies values=map[]\ntime=2026-03-02T17:36:35.752Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies\ntime=2026-03-02T17:36:35.843Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-policies err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace open-cluster-management-policies: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/open-cluster-management-policies?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69a5c675f7e86200018b687e.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69a5c675f7e86200018b687e.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces open-cluster-management-policies)\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c7e81fea097f5360605021ac50258e874a657f0bd852daf57360935530832fa8","failure_pattern":"failed to apply namespace open-cluster-management-policies: an error on the server (\"Internal Server Error: \\\\\"/api/v1/namespaces/open-cluster-management-policies?fieldManager=templatize\u0026amp;force=false\\\\\": Post \\\\\"\u003curl\u003e\": proxy error from localhost:9443","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace open-cluster-management-policies: an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/open-cluster-management-policies?fieldManager=templatize\u0026amp;force=false\\\\\\\": Post \\\\\\\"https://guard.69a5c675f7e86200018b687e.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": proxy error from localhost:9443 while dialing guard.69a5c675f7e86200018b687e.svc.cluster.local:443, code 503: 503 Service Unavailable\\\") has prevented the request from succeeding (patch namespaces open-cluster-management-policies)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c7e81fea097f5360605021ac50258e874a657f0bd852daf57360935530832fa8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4273/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028510182178295808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4273,"pr_state":"closed","pr_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","final_merged_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T16:38:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b991e80e99754940c66498288f2a9c0fdb6000cb8dcd1423b95616eae8921ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4273/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028510182178295808","occurred_at":"2026-03-02T16:38:03Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T17:36:23.915Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-02T17:36:33.264Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=8623c549-1408-41a0-8d82-10ea112d6ede \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=a5ebe6d1-c61a-4580-bf1c-d1e9107c4241 \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj8295808.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj8295808.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j8295808 -n arohcp-prow-dbcs-j8295808 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8295808/providers/Microsoft.Network/dnszones/j8295808.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j8295808-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j8295808-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set global.imageDigest=sha256:877909c7b876e7cb8dd1d0cdb6485b92e9eb0bdd1494df44a308363b7a44b63b \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j8295808-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"ede94721-7d0c-4c4f-b1d8-7323cf9aa5e7\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j8295808-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8295808-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j8295808-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nError from server (InternalError): error when retrieving current configuration of:\\nResource: \\\"/v1, Resource=namespaces\\\", GroupVersionKind: \\\"/v1, Kind=Namespace\\\"\\nName: \\\"clusters-service\\\", Namespace: \\\"\\\"\\nfrom server for: \\\"STDIN\\\": an error on the server (\\\"Internal Server Error: \\\\\\\"/api/v1/namespaces/clusters-service\\\\\\\": Post \\\\\\\"https://guard.69a5c6b6b0620a0001af9d93.svc.cluster.local:443/subjectaccessreviews?timeout=30s\\\\\\\": unexpected EOF\\\") has prevented the request from succeeding (get namespaces clusters-service)\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028499006094577664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4278,"pr_state":"closed","pr_sha":"e8e61389ee1c70440541f005cf5c18265eb93f06","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T15:53:45Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"81bb3fb2ec3e49e628968e4317fb6579409be2e5d368390da18e7a21afad1dd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028499006094577664","occurred_at":"2026-03-02T15:53:45Z","failed_at":"e2e","signature_id":"d328dd6dfb0eb87918ad2257ba860cc8a10d4d3b57fea23a5ad8390a5ece7ba2","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff01c0\u003e: \n failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n {\n msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n err: \u003c*errors.StatusError | 0xc000fae640\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n Reason: \"NotFound\",\n Details: {\n Name: \"aro-hcp-exporter\",\n Group: \"\",\n Kind: \"services\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 404,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:70]: Failed to find exporter pod\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff01c0\u003e: \n failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found\n {\n msg: \"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \\\"aro-hcp-exporter\\\" not found\",\n err: \u003c*errors.StatusError | 0xc000fae640\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"services \\\"aro-hcp-exporter\\\" not found\",\n Reason: \"NotFound\",\n Details: {\n Name: \"aro-hcp-exporter\",\n Group: \"\",\n Kind: \"services\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 404,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","failure_pattern":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found","search_query":"failed to get service aro-hcp-exporter/aro-hcp-exporter: services \"aro-hcp-exporter\" not found"}},{"row_id":"b7fe1abccc1c211da53072f2b083b94591d487491ae0993642c8fe46fed53673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028499006094577664","occurred_at":"2026-03-02T15:53:45Z","failed_at":"e2e","signature_id":"394baea090a51d6a0d3f474b357c6927c3e094299b5c653ff1c0de5099cb79f1","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:227]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00055c880\u003e: \n failed to create SRE breakglass session: expected status 202 Accepted, got 500: {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n {\n msg: \"failed to create SRE breakglass session: expected status 202 Accepted, got 500: {\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\",\n err: \u003c*errors.errorString | 0xc0009ba3e0\u003e{\n s: \"expected status 202 Accepted, got 500: {\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:227]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00055c880\u003e: \n failed to create SRE breakglass session: expected status 202 Accepted, got 500: {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n {\n msg: \"failed to create SRE breakglass session: expected status 202 Accepted, got 500: {\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\",\n err: \u003c*errors.errorString | 0xc0009ba3e0\u003e{\n s: \"expected status 202 Accepted, got 500: {\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6e4ba6e14d1469ce3a16d006bcd659e39b361836e7e3a58c848ded3357b21e9d","failure_pattern":"ERROR CODE: InternalServerError; context","search_query":"failed to create SRE breakglass session: expected status 202 Accepted, got 500: {"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1522e4eb73d7617fab02ecce0b0e63d5927b2f31e32e3ae4d1a1e9c753ee5308","6e4ba6e14d1469ce3a16d006bcd659e39b361836e7e3a58c848ded3357b21e9d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028498753828163584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T15:52:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0104ad9349523bbc7eba38796f04ab15994c0ebdba79613b9fd1f7289f76d693","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028498753828163584","occurred_at":"2026-03-02T15:52:45Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T16:27:52.957Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T16:27:54.887Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T16:27:56.078Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=08405d888910999257cba53f540d9735e3ae4419ccf3ff64a552174fb8847d4c\ntime=2026-03-02T16:29:57.122Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8163584/providers/Microsoft.Resources/deployments/08405d888910999257cba53f540d9735e3ae4419ccf3ff64a552174fb8847d4c/operationStatuses/08584291380101412755\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8163584/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8163584/providers/Microsoft.Monitor/accounts/services-j8163584/providers/Microsoft.Authorization/roleAssignments/910a7fb2-aa24-55c6-b06b-9a768c5b5e02\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T162809Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'c81fe5ce-6901-42d4-a994-1e2082e88dfb', request correlation id 'ec77b5e2-1272-4eb3-a45d-5ce11c44e252'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8163584/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8163584/providers/Microsoft.Monitor/accounts/hcps-j8163584/providers/Microsoft.Authorization/roleAssignments/04d546f8-9e55-5388-aad8-adc7d3d8c7a4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T162853Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '66343657-0aeb-42f7-b04c-86ee9ede5436', request correlation id 'ec77b5e2-1272-4eb3-a45d-5ce11c44e252'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028497903252672512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T15:49:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8fbedd819b6dd4666a9d365bd103591dacb44d62485342c68e264fce3a50728f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028497903252672512","occurred_at":"2026-03-02T15:49:22Z","failed_at":"provision","signature_id":"1f9a53244b8872cc840dcdb9a5340656fe64e74fe875206e25f12672b732ebe1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.AdminAPI/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T16:48:34.001Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T16:48:34.876Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy values=\"map[audit:map[connectSocket:false] azure:map[clientId:f5070068-ce53-445a-9f62-db9f2fd1d415 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service] cosmosDB:map[name:arohcpprow-rp-j2672512 url:https://arohcpprow-rp-j2672512.documents.azure.com:443/] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:admin-api-cert-prow-j2672512] deployment:map[limits:map[cpu:1 memory:1Gi] replicas:2 requests:map[cpu:200m memory:512Mi] zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 keyVaultName:aro-hcp-dev-svc-kv] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:admin-api-https] host:admin.j2672512.hcpsvc.osadev.cloud] image:map[digest:sha256:c6c34fbdb1420e7a73e62bd5e93b840d2cc72e8935a8b8c7ac3fa39b150d06ac registry:arohcpsvcdev.azurecr.io repository:ci-op-34qw519h/pipeline] kusto:map[endpoint:https://hcp-dev-us-2.eastus2.kusto.windows.net] location:westus3 mise:map[deploy:false policyLabel:Geneva Actions] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-34qw519h/pipeline:pull workloadIdentityClientId:69233576-738e-4519-83e8-855e72ed4da9 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:admin-api port:8443] serviceAccount:map[name:admin-api workloadIdentityClientId:4010f6ed-934c-4d39-9eaf-c8bbaea77ecc workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] sessiongate:map[maxSessionTTL:24h minSessionTTL:1m mise:map[policyLabel:Session Gate] namespace:sessiongate servicePort:8080] virtualService:map[host:admin.j2672512.hcpsvc.osadev.cloud]]\"\ntime=2026-03-02T16:48:34.876Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy\ntime=2026-03-02T16:48:35.262Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp-admin-api: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a839b8d340387460e7313de0048647749cd092d8b27686968dcecb1fdf8848b","failure_pattern":"failed to apply namespace aro-hcp-admin-api: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp-admin-api: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8a839b8d340387460e7313de0048647749cd092d8b27686968dcecb1fdf8848b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028496311908569088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T15:42:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2ef30ee5bf9acb7324016fbd495084b0d60da61387bd0a5c05bbbb5b3563fe7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028496311908569088","occurred_at":"2026-03-02T15:42:46Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T16:19:51.549Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T16:19:53.487Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T16:19:54.793Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7437bab299c4fa278476040288aab9e8ba23dfc89c50f96ea08d68cb4b623d5b\ntime=2026-03-02T16:21:25.519Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8569088/providers/Microsoft.Resources/deployments/7437bab299c4fa278476040288aab9e8ba23dfc89c50f96ea08d68cb4b623d5b/operationStatuses/08584291384915051761\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8569088/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8569088/providers/Microsoft.Monitor/accounts/hcps-j8569088/providers/Microsoft.Authorization/roleAssignments/2d108d97-e8f5-522c-9037-c621b0e91e5e\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T162016Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '4f963a38-245a-4100-a542-ee11cac69c06', request correlation id 'ba3dbbd7-23a7-4381-b8d4-c05c016cde74'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028495403820781568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T15:39:26Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"68dd682d9cd548dec14da982d93069f1336e03371d7ecf1c7b6a90615e10d567","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028495403820781568","occurred_at":"2026-03-02T15:39:26Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T16:31:05.473Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T16:31:07.570Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T16:31:08.715Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ee7a10f42c52fc69f4cb5290ab1d31345cf26cea51adec734f97fbd1d1b8419e\ntime=2026-03-02T16:33:09.728Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0781568/providers/Microsoft.Resources/deployments/ee7a10f42c52fc69f4cb5290ab1d31345cf26cea51adec734f97fbd1d1b8419e/operationStatuses/08584291378174645338\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0781568/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0781568/providers/Microsoft.Monitor/accounts/services-j0781568/providers/Microsoft.Authorization/roleAssignments/07949d40-4c05-576c-b392-bddbc6119ce4\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T163132Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '91e9b2ff-85a8-4aa5-9832-fbffcce35deb', request correlation id '2e6e3110-51af-4718-b64b-c023f7ef606d'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0781568/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0781568/providers/Microsoft.Monitor/accounts/hcps-j0781568/providers/Microsoft.Authorization/roleAssignments/4bfffe1d-0ed3-50e9-b4ef-5d3b197f53ea\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T163122Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7cb82cdc-282d-49a5-8712-41ff21a3aaa6', request correlation id '2e6e3110-51af-4718-b64b-c023f7ef606d'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028494593984565248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T15:36:13Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fd7e6ff147498d0af1696fca4ce63deccd245b3c099ca98ccea6cfa8d7f2d1cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028494593984565248","occurred_at":"2026-03-02T15:36:13Z","failed_at":"e2e","signature_id":"104858e24d23a96511f42a804a265755e06e63f503b6e47b66f5ff263c2f5e09","test_name":"Customer should not be able to create a 4.18 HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_missing_info.go:82]: Expected\n \u003c*fmt.wrapError | 0xc000981ba0\u003e: \n failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \"illegal-hcp-cluster\" in resourcegroup=\"illegal-ocp-version-sq6jt2\": http2: client connection lost\n {\n msg: \"failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-sq6jt2\\\": http2: client connection lost\",\n err: \u003c*fmt.wrapError | 0xc000981b80\u003e{\n msg: \"failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-sq6jt2\\\": http2: client connection lost\",\n err: \u003c*errors.errorString | 0xc000c08a70\u003e{\n s: \"http2: client connection lost\",\n },\n },\n }\nto match error\n \u003c*matchers.MatchRegexpMatcher | 0xc0008da3c0\u003e: {\n Regexp: \"Version .* (doesn't exist|is disabled)\",\n Args: nil,\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_missing_info.go:82]: Expected\n \u003c*fmt.wrapError | 0xc000981ba0\u003e: \n failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \"illegal-hcp-cluster\" in resourcegroup=\"illegal-ocp-version-sq6jt2\": http2: client connection lost\n {\n msg: \"failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-sq6jt2\\\": http2: client connection lost\",\n err: \u003c*fmt.wrapError | 0xc000981b80\u003e{\n msg: \"failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-sq6jt2\\\": http2: client connection lost\",\n err: \u003c*errors.errorString | 0xc000c08a70\u003e{\n s: \"http2: client connection lost\",\n },\n },\n }\nto match error\n \u003c*matchers.MatchRegexpMatcher | 0xc0008da3c0\u003e: {\n Regexp: \"Version .* (doesn't exist|is disabled)\",\n Args: nil,\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6c909a60aa3806b40941c3a45de8a216704a5dee354d1a9140ea6ea9e7f0d2d","failure_pattern":"Version .* (doesn't exist|is disabled)","search_query":"Version .* (doesn't exist|is disabled)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6c909a60aa3806b40941c3a45de8a216704a5dee354d1a9140ea6ea9e7f0d2d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028488630351171584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4271,"pr_state":"closed","pr_sha":"70f3b0a98ca89b6e5529c731a62339633818d15e","final_merged_sha":"70f3b0a98ca89b6e5529c731a62339633818d15e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T15:12:31Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"46c9782e65817380ec9feae6a90b9c3ae7b0e31ca4df4abb4b05f35cd01be4ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028488630351171584","occurred_at":"2026-03-02T15:12:31Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T15:56:45.336Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T15:56:47.387Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T15:56:48.635Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9843a4bd43ed96cccdb6dea6f998153214f5a70aeb21cdc7c37463b4a8fecaf9\ntime=2026-03-02T15:58:19.398Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1171584/providers/Microsoft.Resources/deployments/9843a4bd43ed96cccdb6dea6f998153214f5a70aeb21cdc7c37463b4a8fecaf9/operationStatuses/08584291398776320607\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1171584/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1171584/providers/Microsoft.Monitor/accounts/services-j1171584/providers/Microsoft.Authorization/roleAssignments/0f272a2f-0577-5c58-bf97-817cc9e8de28\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T155710Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '4f91f122-0a6e-43ac-84f1-2248ce19271b', request correlation id '669c04fa-79fa-4b66-8716-9b2504d0bfcb'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1171584/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1171584/providers/Microsoft.Monitor/accounts/hcps-j1171584/providers/Microsoft.Authorization/roleAssignments/d9dcf8e0-273a-59af-a015-d61c1a81d415\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T155701Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '1506df82-2838-4d45-82df-3005e1600ac6', request correlation id '669c04fa-79fa-4b66-8716-9b2504d0bfcb'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028487626863939584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4276,"pr_state":"closed","pr_sha":"a75bab5ee1d156ac57ff1f8b7d157b5e9f8a898c","final_merged_sha":"a75bab5ee1d156ac57ff1f8b7d157b5e9f8a898c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T15:08:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028486932408832000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T15:05:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028479808928747520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"bf516b5b5d16ee979b9280f8cafa4976206218ae","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T14:37:28Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e06c238b04664afae7a7c5662337243a732bdc70ad6c7a4a57a1a38419a44cbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028479808928747520","occurred_at":"2026-03-02T14:37:28Z","failed_at":"provision","signature_id":"498357a349faaf4c5a62b430d44c3d84876304b3a08160c1f81f4411e909b30c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T15:44:01.401Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T15:44:02.141Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:7f4521eb-b0a2-4c2c-9ea3-724117af1606 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j8747520] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j8747520.hcpsvc.osadev.cloud] image:map[digest:sha256:63b72342a60be1e013579d1a23c75a9f9d5a92b74950e43e6e41f7af3d1d02df registry:arohcpsvcdev.azurecr.io repository:ci-op-b6rrv4j7/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-b6rrv4j7/pipeline:pull workloadIdentityClientId:9f85b96c-8ebb-4695-852b-fb6033594f21 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:99fe1d77-3437-44f4-b9b5-785bf2751c1d workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-02T15:44:02.141Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-02T15:44:02.429Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-02T15:44:02.742Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T15:44:02.742Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-02T15:44:02.872Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-02T15:44:05.027Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"unable to continue with install: could not get information about the resource Service \\\"sessiongate\\\" in namespace \\\"sessiongate\\\": Unauthorized\"\ntime=2026-03-02T15:44:05.027Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-02T15:44:05.027Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-02T15:44:05.155Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"failed to get status for release sessiongate: release: not found\"\ntime=2026-03-02T15:44:05.155Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-02T15:44:05.155Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource Service \\\"sessiongate\\\" in namespace \\\"sessiongate\\\": Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"486d9b59545617d652c86d03a9e724b943e648fe07dd411f5928ec6f456b82ea","failure_pattern":"unable to continue with install: could not get information about the resource Service \"sessiongate\" in namespace \"\u003cnamespace\u003e\": Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: unable to continue with install: could not get information about the resource Service \\\"sessiongate\\\" in namespace \\\"sessiongate\\\": Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["486d9b59545617d652c86d03a9e724b943e648fe07dd411f5928ec6f456b82ea"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4273/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028467851777544192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4273,"pr_state":"closed","pr_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","final_merged_sha":"0b6227fc7359d04685e6fa9b61c507ba2576a4fc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T13:49:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028463542931296256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4268,"pr_state":"closed","pr_sha":"d33568473c18b91259361e3a42aceed7021cd0ae","final_merged_sha":"d33568473c18b91259361e3a42aceed7021cd0ae","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T13:32:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028461003485745152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4272,"pr_state":"open","pr_sha":"82c3e2c38229b0ac8188d63a2e0274f9724b0edf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T13:22:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"726b691fa02a61cb31ec70860d38b71069c153d1e16e024c0a1c730d6e89fe8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028461003485745152","occurred_at":"2026-03-02T13:22:44Z","failed_at":"provision","signature_id":"cde242a395c38145b285a06a3b88ef3cf91d736e405e25519990be3dc26a7319","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T14:11:53.106Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T14:11:53.760Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5745152.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5745152.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:88391910-c439-4447-9668-f08c0420eda3 databaseName:arohcpprow-rp-j5745152 databaseUrl:https://arohcpprow-rp-j5745152.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-29w18piz/pipeline@sha256:c9f639992a57c052271b71c778330705f918c787db0f73ef3f6cc434b814f909 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-29w18piz/pipeline:pull workloadIdentityClientId:0cc3f31a-e802-4d62-8f66-f8387f7c3f9f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:88391910-c439-4447-9668-f08c0420eda3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T14:11:53.760Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:11:53.976Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-02T14:11:54.303Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T14:11:54.303Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:11:54.430Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:17:01.091Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T14:17:01.091Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:17:01.091Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:17:02.639Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5745152.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5745152.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:88391910-c439-4447-9668-f08c0420eda3 databaseName:arohcpprow-rp-j5745152 databaseUrl:https://arohcpprow-rp-j5745152.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-29w18piz/pipeline@sha256:c9f639992a57c052271b71c778330705f918c787db0f73ef3f6cc434b814f909 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-29w18piz/pipeline:pull workloadIdentityClientId:0cc3f31a-e802-4d62-8f66-f8387f7c3f9f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:88391910-c439-4447-9668-f08c0420eda3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T14:17:02.640Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTT2/TQBDF7/4UI58SyVsloYAU1EMpnFBRBJyKOKx3H8426x1rdp0oVT48cv4QKxg11ze/NzPP3sk8Egkit2IQ6Y6sTjrp0mPE5TNM+uKCncckLlQFHaSvusZfCWuEvRIbbU7y+GdGRJR/Y4+8oNxDW4iCh0mOQydpYbU0TV4cyPs2LVnci+7qC/bObPeU97xRNZI4E1WpzQrBDti/wwjSQnjtLOTB6xg76mhQEbJ2BmqFrVrr1qfBFnvm3hhuQ+q5B9gu10cXrAtVf8zrMY9DepVTqFPIAdMnNJ63NUIa8A19SyOL1vuBBZvWe1We9QvjApDuRyAkZ/RFhCsWPaZ75OASy3XLPnD47apH3fT31C+tQEkbkquhzB651vsvnf36kK3aEhKQED93DzZmO9osIaBuQky6bqhE2gCBRlYndPIon01m79TkjZrMfkxv59Pp/O3tUz6mmxv6L/N+Ppk95eNxtqNndoFWLtg7FwKkd2Yc6Hxd/bO6vKdsR41wVz0vWlDD9gBf16Qgz1VBNWLUVdeSxUKo3PbCW0TzJwAA//+UDIFBDAQAAA==\"\ntime=2026-03-02T14:17:02.642Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:17:02.642Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T14:18:02.645Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T14:18:04.507Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5745152.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5745152.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:88391910-c439-4447-9668-f08c0420eda3 databaseName:arohcpprow-rp-j5745152 databaseUrl:https://arohcpprow-rp-j5745152.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-29w18piz/pipeline@sha256:c9f639992a57c052271b71c778330705f918c787db0f73ef3f6cc434b814f909 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-29w18piz/pipeline:pull workloadIdentityClientId:0cc3f31a-e802-4d62-8f66-f8387f7c3f9f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:88391910-c439-4447-9668-f08c0420eda3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T14:18:04.507Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:18:04.931Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-02T14:18:05.192Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:18:05.192Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:18:05.367Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-02T14:18:06.351Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:18:06.351Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-02T14:18:06.689Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-02T14:18:07.050Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-02T14:18:07.250Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-02T14:18:07.723Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-02T14:18:08.030Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T14:18:08.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T14:18:08.658Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-02T14:18:09.013Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-02T14:18:09.528Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T14:18:09.902Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-02T14:18:10.228Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T14:18:10.556Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T14:18:10.556Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:18:10.684Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-02T14:23:13.808Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T14:23:13.809Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:23:13.809Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:23:18.501Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5745152.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5745152.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:88391910-c439-4447-9668-f08c0420eda3 databaseName:arohcpprow-rp-j5745152 databaseUrl:https://arohcpprow-rp-j5745152.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-29w18piz/pipeline@sha256:c9f639992a57c052271b71c778330705f918c787db0f73ef3f6cc434b814f909 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-29w18piz/pipeline:pull workloadIdentityClientId:0cc3f31a-e802-4d62-8f66-f8387f7c3f9f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:88391910-c439-4447-9668-f08c0420eda3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T14:23:18.502Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTwW4aMRC971eM9gTSOgJSVRFVDmnaU5UKNT2l6sFrv4KD17Mae0FEfHxlEGWFoOH65r3xm/G8wiORIHInBpHuyeqkk649Bly/wqRvLthpTOLCvKI99F03+AdhhbBDYqvNAR7+KoiIyh/s8dkF68K8rKistVkiWOWhLUTBwyTHIZe0sFqYtqz2wmfIyhn0KuogbpDEmXhG9AWt502DkM7ozvAfjMw6788YbDvvVX3ET4VdWrC4N53dz9g7s9mxvOf1wd9/3p0BklsgJGf0yQaumPMZRpBmwitnIY9ex9g3H/e7U0ts1Ep3Pl3e74Mx3O3XddnuI4c/bv6k2/4j+q0TKOlCcg2U2VGu1V5k53PJ8NX38cTBJZZ3vrv4/alYdjUkICF+zQcbiy2tFxBQ9h+TblqqkdZAoIHVCRkelJPR5KMa3arR5Of4w3R8Nx2PXsoh3dzQJc7kdjq+eymHw2JLr+wCLV2w9y4ESC9mHOiYrn6sTvNUbKkVztWj0YpatnvydU0q8jyvqEGMep5bslgI1Zve8BbR/A0AAP//Tl9nggwEAAA=\"\ntime=2026-03-02T14:23:18.506Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:23:18.506Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T14:24:18.515Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T14:24:20.367Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5745152.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5745152.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:88391910-c439-4447-9668-f08c0420eda3 databaseName:arohcpprow-rp-j5745152 databaseUrl:https://arohcpprow-rp-j5745152.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-29w18piz/pipeline@sha256:c9f639992a57c052271b71c778330705f918c787db0f73ef3f6cc434b814f909 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-29w18piz/pipeline:pull workloadIdentityClientId:0cc3f31a-e802-4d62-8f66-f8387f7c3f9f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:88391910-c439-4447-9668-f08c0420eda3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T14:24:20.367Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:24:20.718Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-02T14:24:20.956Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:24:20.956Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:24:21.148Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-02T14:24:22.132Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:24:22.133Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-02T14:24:22.474Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-02T14:24:22.829Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-02T14:24:22.962Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-02T14:24:23.630Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-02T14:24:23.937Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T14:24:24.237Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T14:24:24.597Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-02T14:24:24.953Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-02T14:24:25.374Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T14:24:25.773Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-02T14:24:26.152Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T14:24:26.476Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T14:24:26.476Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:24:26.662Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-02T14:29:29.772Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T14:29:29.772Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:29:29.772Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:29:33.634Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj5745152.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj5745152.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:88391910-c439-4447-9668-f08c0420eda3 databaseName:arohcpprow-rp-j5745152 databaseUrl:https://arohcpprow-rp-j5745152.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-29w18piz/pipeline@sha256:c9f639992a57c052271b71c778330705f918c787db0f73ef3f6cc434b814f909 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-29w18piz/pipeline:pull workloadIdentityClientId:0cc3f31a-e802-4d62-8f66-f8387f7c3f9f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:88391910-c439-4447-9668-f08c0420eda3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T14:29:33.635Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQW8aMRCF7/srRnsCaR1RiCKVKoc07alKhdqeUvXgtV/BwetZjb0gIn58ZQhlhaDl+vjew/Nmp/BIJIjciUGke7I66aRrjwHXLzDpiwt2GpO4MK9oL33VDf5KWCHslNhqc5CHPwsiovITWs+bBiGVFZVaWC1Mq2ptlgi2J5XVnv8OI0gz4ZWzkEevY8zUm0FFyMoZqCU2aqU7n85EfGOPLHtoC1HwMMlxOEM+GJl13n90wbow7/9P23mv6qN+auzSgsW96hw8Y+/MZkd5z2vVIIkz8R9DzgDJEQjJGX3yuIPvkHO2pF0LTxxcYrm22J3nwRju9su4zD5y+O3mT7rtV6JfO4GSLiTXQJkdcq33Ip13dab+/6/ubZwreyt+fSiWXQ0JSIif8wcbiy2tFxBQnicm3bRUI62BQAOrE7I8KMej8Z0aTdRo/OPd7XR8Ox3fPZdDurmhi8z76WTyXA6HxZZe2AVaumDvXQiQ3plxoON19c/q9J6KLbXC+dfjQytq2e7h60Iq8jyvqEGMep4jWSyE6k1veIto/gQAAP//4VKA0AwEAAA=\"\ntime=2026-03-02T14:29:33.637Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T14:29:33.637Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T14:30:33.641Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028456686431244288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"open","pr_sha":"c853e9af1512d4f8471ce2fa95b25b1ce6f48985","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T13:05:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"96c52556c234bdc9641294998094ac45221168bd26e9df6ff89536054e49d327","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028456686431244288","occurred_at":"2026-03-02T13:05:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T13:40:43.018Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T13:40:45.069Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T13:40:46.418Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7300e993b7cbfcfb27439802b0b649d67a17f09cf9dcb5b040c8707ec18ff524\ntime=2026-03-02T13:42:17.191Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1244288/providers/Microsoft.Resources/deployments/7300e993b7cbfcfb27439802b0b649d67a17f09cf9dcb5b040c8707ec18ff524/operationStatuses/08584291480399328583\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1244288/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1244288/providers/Microsoft.Monitor/accounts/services-j1244288/providers/Microsoft.Authorization/roleAssignments/9fdb02e3-3b6c-5145-b434-458cf72c145b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T134105Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'ddb73d4b-e7db-4012-8bd8-6e62d8605c3c', request correlation id '760873dc-3f53-46b3-a0e7-d9099c672157'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028456529039986688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4254,"pr_state":"closed","pr_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","final_merged_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T13:04:50Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4181dfd1b828ed3bacb0bb7f81ad44fbbe8865fd5a86ea960323d546a865732e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028456529039986688","occurred_at":"2026-03-02T13:04:50Z","failed_at":"provision","signature_id":"f97be132e2abc5b9e1ed32bd4b0708f406b1ea0759d6a477936cca5d5a60792b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-02T14:08:34.560Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-02T14:08:57.180Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.mGG7TxAB4v/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-452gtqdg/pipeline@sha256:cc582d9e2a931e983b0e31b8e7aa1d79fa7bacc19a533c61033dc88541a7aeeb to arohcpsvcdev.azurecr.io/ci-op-452gtqdg/pipeline:cc582d9e2a931e983b0e31b8e7aa1d79fa7bacc19a533c61033dc88541a7aeeb.\\nThe image will still be available under it's original digest sha256:cc582d9e2a931e983b0e31b8e7aa1d79fa7bacc19a533c61033dc88541a7aeeb in the target registry.\\nError: Put \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-452gtqdg/pipeline/blobs/uploads/434d6f6d-411f-4b35-a92a-b660f267d271?_nouploadcache=false\u0026_state=-OTWwXvmUN8mW7YHh8rh_i3MWGuE0akpRlCjGE7zefZ7Ik5hbWUiOiJjaS1vcC00NTJndHFkZy9waXBlbGluZSIsIlVVSUQiOiI0MzRkNmY2ZC00MTFmLTRiMzUtYTkyYS1iNjYwZjI2N2QyNzEiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMDJUMTQ6MDg6MzYuOTI0OTcwMzUyWiJ9\u0026digest=sha256%3Ad6f922f5b32836446d94dbb2e46cd5d47628d4910ae9760e665af4ed57169533\\\": unexpected EOF\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"54d3a0c2e5dc81eed9bb41d33f4edbd4e7b48251b246de62366f64fc56922269","failure_pattern":"Error: Put \"\u003curl\u003e\": unexpected EOF","search_query":"Error: Put \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-452gtqdg/pipeline/blobs/uploads/434d6f6d-411f-4b35-a92a-b660f267d271?_nouploadcache=false\u0026_state=-OTWwXvmUN8mW7YHh8rh_i3MWGuE0akpRlCjGE7zefZ7Ik5hbWUiOiJjaS1vcC00NTJndHFkZy9waXBlbGluZSIsIlVVSUQiOiI0MzRkNmY2ZC00MTFmLTRiMzUtYTkyYS1iNjYwZjI2N2QyNzEiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMDJUMTQ6MDg6MzYuOTI0OTcwMzUyWiJ9\u0026digest=sha256%3Ad6f922f5b32836446d94dbb2e46cd5d47628d4910ae9760e665af4ed57169533\\\": unexpected EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["54d3a0c2e5dc81eed9bb41d33f4edbd4e7b48251b246de62366f64fc56922269"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028455703827451904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T13:01:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6cb871d513de5dc6de186eb54e3ac7a12061e9573d8d398c5b645d8f924a28bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028455703827451904","occurred_at":"2026-03-02T13:01:41Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T13:41:27.096Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T13:41:29.324Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T13:41:30.569Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=61ae1c4bd1ec7a6c2f0eba134e54daca22f6b82a9516c9aa3123b84b9514ba6d\ntime=2026-03-02T13:43:01.342Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7451904/providers/Microsoft.Resources/deployments/61ae1c4bd1ec7a6c2f0eba134e54daca22f6b82a9516c9aa3123b84b9514ba6d/operationStatuses/08584291479956260935\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7451904/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7451904/providers/Microsoft.Monitor/accounts/services-j7451904/providers/Microsoft.Authorization/roleAssignments/5cfaed70-5132-5f60-8641-16a5a6e6deb7\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T134201Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '56a4f011-75c3-4f51-b76c-585354a7d1b6', request correlation id '7fc76d25-3642-40e2-82c3-439145ef7b5e'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028454972831567872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T12:58:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028454161003057152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T12:55:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028453385421721600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T12:52:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4154/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028453287669272576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4154,"pr_state":"closed","pr_sha":"5e86cd02a0c8a7135ec82a4b7d287c8a3f9fd8a1","final_merged_sha":"87ae44a893e18a3f688b7a042698f29e92dcdb8d","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-02T12:52:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028447002865438720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4271,"pr_state":"closed","pr_sha":"70f3b0a98ca89b6e5529c731a62339633818d15e","final_merged_sha":"70f3b0a98ca89b6e5529c731a62339633818d15e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T12:27:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ce05c040d1cd310f5ed89dff9bf007df1e537f235c94ea323b47e3f6af03d01d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028447002865438720","occurred_at":"2026-03-02T12:27:06Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T13:04:05.111Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T13:04:07.145Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T13:04:08.522Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f430010b918596f0b6c17d46d83e01316a73c6cc5c6fbf7277b1ce6ab540fa48\ntime=2026-03-02T13:06:09.597Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5438720/providers/Microsoft.Resources/deployments/f430010b918596f0b6c17d46d83e01316a73c6cc5c6fbf7277b1ce6ab540fa48/operationStatuses/08584291502378504020\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5438720/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5438720/providers/Microsoft.Monitor/accounts/services-j5438720/providers/Microsoft.Authorization/roleAssignments/fea6b229-f784-5f6f-85d9-5d41e1ac3bc9\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T130422Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f948492a-1b4f-46a8-9fe8-7c12deb697e6', request correlation id 'd1b2a7f6-127b-48d4-908d-5829b896234f'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5438720/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5438720/providers/Microsoft.Monitor/accounts/hcps-j5438720/providers/Microsoft.Authorization/roleAssignments/ba5500cc-d533-56a4-b8ad-06b246d1efb0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T130430Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6418c325-81e0-4b7f-a31b-6e5da23cb09d', request correlation id 'd1b2a7f6-127b-48d4-908d-5829b896234f'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028442454805450752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T12:09:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"177eb8b148e9301e3459bde8edd724dc58483bf1398e70a3416ceafc70f0df88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028442454805450752","occurred_at":"2026-03-02T12:09:02Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T12:36:33.017Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T12:36:34.621Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T12:36:35.753Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=69cc773df89681e68aab3ee0dd1bc114c934fdea64745c4f38d480efac07c518\ntime=2026-03-02T12:38:36.737Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5450752/providers/Microsoft.Resources/deployments/69cc773df89681e68aab3ee0dd1bc114c934fdea64745c4f38d480efac07c518/operationStatuses/08584291518903944627\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5450752/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5450752/providers/Microsoft.Monitor/accounts/hcps-j5450752/providers/Microsoft.Authorization/roleAssignments/d3403ecf-c41d-55ec-8c17-eb6fac690d7b\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T123727Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '402adf5f-ccf8-482e-bebd-a85c251b8fdc', request correlation id '6cc58b00-2384-45d1-adf5-b7b8b9189692'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028440113955999744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"35f2d8f4e5341e8214b950098a9a81bcf4557ea3","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-02T11:59:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028435970176585728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T11:43:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4270/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028427960758833152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4270,"pr_state":"closed","pr_sha":"3537ac4cb8799928ae35da15407a57382259364d","final_merged_sha":"3537ac4cb8799928ae35da15407a57382259364d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T11:11:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028422454333935616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4205,"pr_state":"closed","pr_sha":"9d1190fce2222e51aba585b4db483448a7116984","final_merged_sha":"a36f37aae6ead63c15a10186a904f97e4e0fc50d","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-02T10:49:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028421833165901824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4269,"pr_state":"closed","pr_sha":"1ef54a64159bceca800923cb4087bf092c55a969","final_merged_sha":"1ef54a64159bceca800923cb4087bf092c55a969","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T10:47:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028420873341374464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4268,"pr_state":"closed","pr_sha":"d33568473c18b91259361e3a42aceed7021cd0ae","final_merged_sha":"d33568473c18b91259361e3a42aceed7021cd0ae","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T10:43:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028419171179565056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T10:36:31Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"19b3f9f5e2a4c56ed15255e858be60afdae907aa895957336e1e55131cc822f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028419171179565056","occurred_at":"2026-03-02T10:36:31Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T11:07:37.314Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T11:07:39.199Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T11:07:40.736Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=684e01ed42e1f4d3151491f3c9c00065b1edb8a507128e01bb5f0d946ab45dee\ntime=2026-03-02T11:09:11.564Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9565056/providers/Microsoft.Resources/deployments/684e01ed42e1f4d3151491f3c9c00065b1edb8a507128e01bb5f0d946ab45dee/operationStatuses/08584291572256880052\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9565056/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9565056/providers/Microsoft.Monitor/accounts/services-j9565056/providers/Microsoft.Authorization/roleAssignments/1aa3a540-ff87-5da7-ac6e-b161c3880fb7\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T110803Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '386331d9-ad2b-470c-828b-0466c50ff8ea', request correlation id '334d8f09-11b6-4983-a51c-6f3e50d0e7ad'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028413549126094848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"b95c63b18fc9e7a6b452f47f281c94edf7aec077","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T10:14:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4207f144f46a5178b25da89f1db4b4423a9a78f1b8771cd2cef2928374ce17e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028413549126094848","occurred_at":"2026-03-02T10:14:10Z","failed_at":"provision","signature_id":"b1174a995fd8d6059e31e11cbbcb96d9ae17b651be2c55f66f9fc06a54a62a4e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-02T11:03:16.467Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T11:03:17.161Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6094848.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6094848.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 databaseName:arohcpprow-rp-j6094848 databaseUrl:https://arohcpprow-rp-j6094848.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h9mmb6hh/pipeline@sha256:1f4f5d800ad379a1e1e5b4857ac913affba44cca632c69413822a278936d5cb9 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h9mmb6hh/pipeline:pull workloadIdentityClientId:a40b2693-1a01-4f89-9a57-94132413f653 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T11:03:17.161Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:03:17.225Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-02T11:03:17.341Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T11:03:17.341Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:03:17.454Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:08:23.304Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T11:08:23.304Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:08:23.304Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:08:27.600Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6094848.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6094848.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 databaseName:arohcpprow-rp-j6094848 databaseUrl:https://arohcpprow-rp-j6094848.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h9mmb6hh/pipeline@sha256:1f4f5d800ad379a1e1e5b4857ac913affba44cca632c69413822a278936d5cb9 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h9mmb6hh/pipeline:pull workloadIdentityClientId:a40b2693-1a01-4f89-9a57-94132413f653 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T11:08:27.601Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5RTQW8aPRS876942hNI6wiI9H0VVQ5p2lOVCjU9terBa0/Bweu3evaCiPjxlSEpK7Sk6XU882bGeq/wSCSI3IlBpBuyOumka48R148w6bMLdh6TuLCs6Ah90Q3+QNggHJDYavMCj38URETlA4wgLYQ3zkLuvI6xrKistVkjWBUhG2eg1tipje58yo9aWK1MW1bHEXccfrnlvW77Qv3UCZR0IbkGyhwob9VeZH9ljwx7aAtR8DDJcRhgfkTredcg9AOrZ4MB/sOx5z0Hl1j+SXNrDHdHn8vcnPyDC9aFZb/p34s8mwwkUg2SOBMHRLdGFp33A4Zt572qT/i5sEsrFvekc5oFe2d2B5b3vH3xe+VHFoDkEQjJGX3W6LXcxc/3xbqrIQEJ8VNe2FjsabuCgPIGxaSblmqkLRBoZHVChkflbDL7T02u1WT2bTqdT67n0/+/l2O6uqKLnHfzWeaMiz09sgu0dsHeuBAgvTPjQKfr6p/V+T0Ve2qF8+spaEUt2yP5bUMq8rysqEGMeplHslgI1bteeYtofgcAAP//F0qFBgwEAAA=\"\ntime=2026-03-02T11:08:27.603Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:08:27.603Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-02T11:09:27.605Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T11:09:28.245Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6094848.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6094848.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 databaseName:arohcpprow-rp-j6094848 databaseUrl:https://arohcpprow-rp-j6094848.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h9mmb6hh/pipeline@sha256:1f4f5d800ad379a1e1e5b4857ac913affba44cca632c69413822a278936d5cb9 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h9mmb6hh/pipeline:pull workloadIdentityClientId:a40b2693-1a01-4f89-9a57-94132413f653 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T11:09:28.245Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:09:28.603Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-02T11:09:28.839Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:09:28.839Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:09:29.110Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-02T11:09:30.108Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:09:30.108Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-02T11:09:30.447Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-02T11:09:30.900Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-02T11:09:31.088Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-02T11:09:31.580Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-02T11:09:31.910Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T11:09:32.266Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T11:09:32.606Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-02T11:09:32.915Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-02T11:09:33.627Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T11:09:34.009Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-02T11:09:34.391Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T11:09:34.692Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T11:09:34.692Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:09:34.820Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-03-02T11:14:37.800Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-02T11:14:37.800Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:14:37.800Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:14:41.903Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6094848.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6094848.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 databaseName:arohcpprow-rp-j6094848 databaseUrl:https://arohcpprow-rp-j6094848.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h9mmb6hh/pipeline@sha256:1f4f5d800ad379a1e1e5b4857ac913affba44cca632c69413822a278936d5cb9 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h9mmb6hh/pipeline:pull workloadIdentityClientId:a40b2693-1a01-4f89-9a57-94132413f653 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T11:14:41.905Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5STQW/aQBCF7/4VI5+C5I2AoEqlyiFNe6pSoaanVD2sd19hw3rHml2DiPjxle1SLARNerPefu95Zmcn80gkiNyIQaRbsjrppEuPKy6fYdIXF+w8JnFhWVAvfdUV/krYIHRKrLU5yKMfGRFR/gjZOIM7Y7gJKS8oL7VZI9j2UwurlanzomfvOfxyywddDzClXxqBkiYkV0GZDnmr9yL9p6rBiTp4KiRxJp4xfULteVehb+PEd4a/M7JovP/ognVhOayrbrxX5VE/NTZpxeJedHIcFuyd2XWU97w91PeP/y4AaSMQkjNdxv/1+Y19dzMe2kIUPMxJyJA8097rxkcYQVoIb5yF3Hsd4zAg9uNRa+zURjc+XR7hAweXWF6ZSPbzQ7ZuSkhAQvzcPtiY7Wm7goDahxWTrmoqkbZAoCurE1r5Kp+Op+/U+EaNp98nk/n4/fxm9pSP6PqaLjGT2Xw2ecpHo2xPz+wCrV2wty4EyGDNONBxu4ZrdbpP2Z5q4fb0WGhBNdsefltIQZ6XBVWIUS/bSBYLoXI3aN4imt8BAAD//2OiqtcMBAAA\"\ntime=2026-03-02T11:14:41.931Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:14:41.931Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-02T11:15:41.934Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-02T11:15:42.820Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6094848.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6094848.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 databaseName:arohcpprow-rp-j6094848 databaseUrl:https://arohcpprow-rp-j6094848.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h9mmb6hh/pipeline@sha256:1f4f5d800ad379a1e1e5b4857ac913affba44cca632c69413822a278936d5cb9 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h9mmb6hh/pipeline:pull workloadIdentityClientId:a40b2693-1a01-4f89-9a57-94132413f653 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T11:15:42.820Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:15:43.125Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-02T11:15:43.367Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:15:43.367Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:15:43.582Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-02T11:15:44.591Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:15:44.591Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-03-02T11:15:44.906Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-03-02T11:15:45.268Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-03-02T11:15:45.521Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-03-02T11:15:46.028Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-03-02T11:15:46.352Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T11:15:46.676Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T11:15:47.029Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-03-02T11:15:47.351Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-03-02T11:15:47.738Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-03-02T11:15:48.116Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-03-02T11:15:48.455Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-03-02T11:15:48.759Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-02T11:15:48.759Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:15:48.931Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-03-02T11:20:51.941Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-02T11:20:51.941Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:20:51.941Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:20:56.605Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6094848.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6094848.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 databaseName:arohcpprow-rp-j6094848 databaseUrl:https://arohcpprow-rp-j6094848.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-h9mmb6hh/pipeline@sha256:1f4f5d800ad379a1e1e5b4857ac913affba44cca632c69413822a278936d5cb9 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-h9mmb6hh/pipeline:pull workloadIdentityClientId:a40b2693-1a01-4f89-9a57-94132413f653 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:aaeb6c8a-255a-4ec3-827e-1aacc19620f8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-02T11:20:56.606Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5RTQW7bMBC86xULnWxADGy3CQoXOaRpT0UKo+kpRQ8UObUZU1xhSdlw4McXsuJaMKImvRHD2dlZcifzSCSI3IhBpGuyOumkS48Rl48w6asLdh6TuLAsqIO+6Qp/IWwQDkistTnC458ZEVF+D9k4gxtjuAkpLygvtVkj2PaohdXK1HnRcW85/HbLO133aEo/NQIlTUiugjIHyltrB9nf2aOFPbSFKHiY5Di8wPyM2vOuQuf9+VINz3BjZNF4/8kF68Kyb6ZuvFflCT8vbNKKxT3p1siCvTO7A8t73qoKSZyJ/+i7AKSVQEjO6LNhjnVHnYEXecH06w/0/MH/1+0eRpAWwhtnIbdex9jvGjtNtcZObXTj03DfOw4usbzyOdmvj9m6KSEBCfFLu7Ax29N2BQG1ixWTrmoqkbZAoJHVCS08ymeT2ZWavFOT2Y/pdD69nL//8JCP6eKChjizyfzy6iEfj7M9PbILtHbBXrsQIL2YcaBTuvqxOs9TtqdauL09GS2oZtuR3yZSkOdlQRVi1MtWksVCqNz1hreI5k8AAAD//+8o5CsMBAAA\"\ntime=2026-03-02T11:20:56.608Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-02T11:20:56.608Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-03-02T11:21:56.608Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4267/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028403861902856192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4267,"pr_state":"closed","pr_sha":"110207af43a2d535cf2a71c1e17345d2a74e3d5f","final_merged_sha":"110207af43a2d535cf2a71c1e17345d2a74e3d5f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T09:35:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028384022798798848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"f743ff17fb2145a02ae8d79d65f69382706ab53d","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T08:16:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c40bb9fdd743c3664c351b51cae0a57e3813039215860d709b0e82a3d97301fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028384022798798848","occurred_at":"2026-03-02T08:16:51Z","failed_at":"provision","signature_id":"7acb328ca571196613fcc24303d5ec48dcd52f7ac5349cdf5ee6de30f5b52e6c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/kustoDataConnection","test_suite":"step graph","failure_text":"time=2026-03-02T08:45:42.986Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection description=\"Step kustoDataConnection\\n Kind: ARM\\n Template: templates/audit-logs-data-connection.bicep\\n Parameters: configurations/audit-logs-data-connection.tmpl.bicepparam\"\ntime=2026-03-02T08:45:43.703Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection\ntime=2026-03-02T08:45:44.300Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection deployment=307e4852c7530fe12ab3defcb5fbfa3219961982d2c9229ff0c241ca5c26ccc4\ntime=2026-03-02T08:46:14.383Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/307e4852c7530fe12ab3defcb5fbfa3219961982d2c9229ff0c241ca5c26ccc4/operationStatuses/08584291657414570416\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CanNotAlterEventHubResourceId\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0241280/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8798848/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d75be1569a2db42bca9087fe5a7bd21fee0536b43d9658da65baebfa84fcd3dd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CanNotAlterEventHubResourceId; detail message [BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs; provider Microsoft.EventHub","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d75be1569a2db42bca9087fe5a7bd21fee0536b43d9658da65baebfa84fcd3dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028375309522309120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4128,"pr_state":"closed","pr_sha":"97dcd3444b49cca0f791a06d67de76d5a3fc1f4b","final_merged_sha":"97dcd3444b49cca0f791a06d67de76d5a3fc1f4b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T07:42:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4152/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028370323451154432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4152,"pr_state":"closed","pr_sha":"2f8a0eb8477e5204a65847041c1967c2843f8f80","final_merged_sha":"2f8a0eb8477e5204a65847041c1967c2843f8f80","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-02T07:22:24Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f6cce7da4c32b64b65d8a04794c55c281616bb7a39098bff111d113f2a61a710","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4152/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028370323451154432","occurred_at":"2026-03-02T07:22:24Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T07:52:18.654Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T07:52:20.674Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T07:52:22.078Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=07c2142ea0cf364bd52754d7f0b056a76996857e8c23bcf2b8e0a7f71c8ad27c\ntime=2026-03-02T07:54:22.812Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1154432/providers/Microsoft.Resources/deployments/07c2142ea0cf364bd52754d7f0b056a76996857e8c23bcf2b8e0a7f71c8ad27c/operationStatuses/08584291689441883780\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1154432/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1154432/providers/Microsoft.Monitor/accounts/services-j1154432/providers/Microsoft.Authorization/roleAssignments/83ed6509-5ddb-56a0-8ee1-9be4f91fcaff\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T075307Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '62b7c5b6-6b9e-4982-ab8e-7240b3bccd9a', request correlation id 'aec418ff-70ff-4bc8-85f5-722dc7b6bbd9'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4249,"pr_state":"closed","pr_sha":"67bc090ef1f250bde852b94552a8cfc7fe13e836","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T07:10:51Z"},"failed_at":["e2e"],"failed_test_count":17,"occurrences":[{"row_id":"0cfb5aea47b1cc90695103f0193b37666eb170b83fb7c4283c3dc41195916490","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"49c08a27f9463fe4be563555377f8dd6c546b955f2c4d4d4774c212f817f2484","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b29080\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-rnqjn5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rnqjn5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011f2d80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5\",\n },\n \u003c*fmt.wrapErrors | 0xc000b29050\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rnqjn5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011f2d80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b29080\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-rnqjn5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rnqjn5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011f2d80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5\",\n },\n \u003c*fmt.wrapErrors | 0xc000b29050\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-rnqjn5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011f2d80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-rnqjn5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"19bb1c39999935ca396834103c6683dfb053858981a9ab4c1ffa92638df2f722","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"b64f64608f28128536f83444d8eb64efd1cddca70c807a4817e225a93ddaa761","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010e2210\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ks7gck\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ks7gck\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010b8460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck\",\n },\n \u003c*fmt.wrapErrors | 0xc0010e21b0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ks7gck\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010b8460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010e2210\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ks7gck\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ks7gck\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010b8460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck\",\n },\n \u003c*fmt.wrapErrors | 0xc0010e21b0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-ks7gck\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010b8460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-ks7gck\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"3a2f5856db931e73ba232d91749902b17a5338df8abf6f2ea2da47d3ed2d27eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"53012fc64d564a6e4612d11926dc1b1a233d3187ddc4d9704a0592862f7dfb62","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"er.go:284\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f4408da0e18, 0xc000f719e0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000f719e0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/02/26 09:38:24.47\n \"ts\"=\"2026-03-02 09:38:24.471074\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-02 09:38:24.471141\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b9bsbq\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\n \"ts\"=\"2026-03-02 09:38:25.297342\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-b9bsbq\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 09:38:25.948355\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-hcj7nb\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\n \"ts\"=\"2026-03-02 09:38:26.124631\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-hcj7nb\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 09:38:26.806953\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:26.807052\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:26.807115\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:26.808108\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\"\n \"ts\"=\"2026-03-02 09:38:26.808138\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\n \"ts\"=\"2026-03-02 09:38:26.808150\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-36\"\n \"ts\"=\"2026-03-02 09:38:26.808117\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-117\"\n \"ts\"=\"2026-03-02 09:38:27.376106\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:27.819530\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-02 09:38:28.214855\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:28.612349\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:29.001198\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"msi-deployment-73\"\n \"ts\"=\"2026-03-02 09:38:29.353261\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-hcj7nb\"\n \"ts\"=\"2026-03-02 09:38:29.690122\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-36\" \"deployment\"=\"msi-deployment-36\"\n \"ts\"=\"2026-03-02 09:38:30.100969\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-117\" \"deployment\"=\"msi-deployment-117\"\n \"ts\"=\"2026-03-02 09:38:30.810764\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-02 09:38:31.926618\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b9bsbq\"\n \"ts\"=\"2026-03-02 09:38:32.387683\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:32.387805\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-02 09:38:32.387904\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ner.go:284\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f4408da0e18, 0xc000f719e0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000f719e0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/02/26 09:38:24.47\n \"ts\"=\"2026-03-02 09:38:24.471074\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-02 09:38:24.471141\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b9bsbq\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\n \"ts\"=\"2026-03-02 09:38:25.297342\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-b9bsbq\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 09:38:25.948355\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-hcj7nb\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\n \"ts\"=\"2026-03-02 09:38:26.124631\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-hcj7nb\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 09:38:26.806953\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:26.807052\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:26.807115\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:26.808108\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\"\n \"ts\"=\"2026-03-02 09:38:26.808138\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\n \"ts\"=\"2026-03-02 09:38:26.808150\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-36\"\n \"ts\"=\"2026-03-02 09:38:26.808117\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-117\"\n \"ts\"=\"2026-03-02 09:38:27.376106\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:27.819530\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-02 09:38:28.214855\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:28.612349\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-117\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:29.001198\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-73\" \"deployment\"=\"msi-deployment-73\"\n \"ts\"=\"2026-03-02 09:38:29.353261\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-hcj7nb\"\n \"ts\"=\"2026-03-02 09:38:29.690122\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-36\" \"deployment\"=\"msi-deployment-36\"\n \"ts\"=\"2026-03-02 09:38:30.100969\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-117\" \"deployment\"=\"msi-deployment-117\"\n \"ts\"=\"2026-03-02 09:38:30.810764\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-02 09:38:31.926618\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-sr4rrn\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b9bsbq\"\n \"ts\"=\"2026-03-02 09:38:32.387683\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:32.387805\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-02 09:38:32.387904\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-sr4rrn\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"3fee8ac8cfc5c4f85b96a9d0ee701c19aed16a2fa8637a534bf537cfabf93137","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"52cb8a6a8116e757bb10015fe5810381baa88301d8a7ddfd4a2bff34506020c2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014db6e0\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nsmfpf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-nsmfpf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000bf3a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf\",\n },\n \u003c*fmt.wrapErrors | 0xc0014db6b0\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-nsmfpf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000bf3a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014db6e0\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nsmfpf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-nsmfpf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000bf3a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf\",\n },\n \u003c*fmt.wrapErrors | 0xc0014db6b0\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-nsmfpf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000bf3a60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-nsmfpf\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"56f7d1e57c10c7d4dfc2d5477b078f70d84057aaae2e69d1415c42d9a6f45c14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"cbc6c94c9084115bbf8c9b0b9e5a75c779d32d703b494b10bf1f4b4ade467a3f","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001387080\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-np75j2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-np75j2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001254db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2\",\n },\n \u003c*fmt.wrapErrors | 0xc001387050\u003e{\n msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-np75j2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001254db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001387080\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-np75j2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-np75j2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001254db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2\",\n },\n \u003c*fmt.wrapErrors | 0xc001387050\u003e{\n msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-np75j2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001254db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-np75j2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"622616ca993a82fd282c3b5e7718efe36e37ae9dd956d353a2e653c9558a04bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"cb07408034962413dcfd07c06618a03dfb17213e451348f60c0716b4f0f9dd53","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000fef050\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-hmw7cp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-hmw7cp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fbc5b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp\",\n },\n \u003c*fmt.wrapErrors | 0xc000fef020\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-hmw7cp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fbc5b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000fef050\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-hmw7cp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-hmw7cp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fbc5b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp\",\n },\n \u003c*fmt.wrapErrors | 0xc000fef020\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-hmw7cp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fbc5b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-hmw7cp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"6f58f6b8d8b2197092b2198542f7768c55961e5bf0e7e8472af54824690c0b31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"1ca5fa5681476c84897a7d67e70d6e6582da33a368ffa4279137cc6cd68e3fb7","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51 @ 03/02/26 09:38:24.455\n ------------------------------\n Interrupted by User\n First interrupt received; Ginkgo will run any cleanup and reporting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle (Spec Runtime: 1h29m59.294s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51\n In [It] (Node Runtime: 1h29m59.294s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51\n At [By Step] verifying admin credentials request fails with HTTP 409 CONFLICT on deploying cluster (Step Runtime: 35m59.786s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:136\n\n Spec Goroutine\n goroutine 113 [select]\n github.com/onsi/gomega/internal.(*AsyncAssertion).match(0xc0006dd730, {0x81d33b0, 0xc00107df70}, 0x1, {0xc001174000, 0x1, 0x1})\n /go/pkg/mod/github.com/onsi/gomega@v1.39.1/internal/async_assertion.go:558\n github.com/onsi/gomega/internal.(*AsyncAssertion).Should(0xc0006dd730, {0x81d33b0, 0xc00107df70}, {0xc001174000, 0x1, 0x1})\n /go/pkg/mod/github.com/onsi/gomega@v1.39.1/internal/async_assertion.go:145\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func3.1({0x7fbc4048ec20, 0xc000e81260})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158\n | \t// Continue waiting\n | \treturn false\n \u003e }, 45*time.Minute, 30*time.Second).Should(BeTrue(), \"Cluster should become ready within 45 minutes\")\n | \n | // Store all admin credentials for later validation\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000e81260?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 111\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:115 @ 03/02/26 09:38:34.494\n \"ts\"=\"2026-03-02 09:38:34.494685\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-02 09:38:34.494753\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-gzjkx6-7znljn\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\"\n \"ts\"=\"2026-03-02 09:38:35.083189\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-admin-cred-lifecycle-gzjkx6-7znljn\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 09:38:35.622223\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:35.622324\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:35.622380\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:35.623359\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-298\"\n \"ts\"=\"2026-03-02 09:38:35.623380\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\"\n \"ts\"=\"2026-03-02 09:38:36.174760\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-298\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:36.616130\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\" \"deployment\"=\"pooledNonMsiScopedAssignments-6h4svl2qyxdco\"\n \"ts\"=\"2026-03-02 09:38:36.835515\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-298\" \"deployment\"=\"msi-deployment-298\"\n \"ts\"=\"2026-03-02 09:38:37.270141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-gzjkx6-7znljn\"\n \"ts\"=\"2026-03-02 09:38:38.035630\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:38.035751\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-02 09:38:38.035883\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51 @ 03/02/26 09:38:24.455\n ------------------------------\n Interrupted by User\n First interrupt received; Ginkgo will run any cleanup and reporting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle (Spec Runtime: 1h29m59.294s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51\n In [It] (Node Runtime: 1h29m59.294s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51\n At [By Step] verifying admin credentials request fails with HTTP 409 CONFLICT on deploying cluster (Step Runtime: 35m59.786s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:136\n\n Spec Goroutine\n goroutine 113 [select]\n github.com/onsi/gomega/internal.(*AsyncAssertion).match(0xc0006dd730, {0x81d33b0, 0xc00107df70}, 0x1, {0xc001174000, 0x1, 0x1})\n /go/pkg/mod/github.com/onsi/gomega@v1.39.1/internal/async_assertion.go:558\n github.com/onsi/gomega/internal.(*AsyncAssertion).Should(0xc0006dd730, {0x81d33b0, 0xc00107df70}, {0xc001174000, 0x1, 0x1})\n /go/pkg/mod/github.com/onsi/gomega@v1.39.1/internal/async_assertion.go:145\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func3.1({0x7fbc4048ec20, 0xc000e81260})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158\n | \t// Continue waiting\n | \treturn false\n \u003e }, 45*time.Minute, 30*time.Second).Should(BeTrue(), \"Cluster should become ready within 45 minutes\")\n | \n | // Store all admin credentials for later validation\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000e81260?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 111\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:115 @ 03/02/26 09:38:34.494\n \"ts\"=\"2026-03-02 09:38:34.494685\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-02 09:38:34.494753\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-gzjkx6-7znljn\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\"\n \"ts\"=\"2026-03-02 09:38:35.083189\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-admin-cred-lifecycle-gzjkx6-7znljn\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 09:38:35.622223\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:35.622324\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-1fc3cb04d84cb6ff56f76d767f6559b3ce3af645ad2734354d7330138cf8842a.yaml.gz\"\n \"ts\"=\"2026-03-02 09:38:35.622380\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:35.623359\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-298\"\n \"ts\"=\"2026-03-02 09:38:35.623380\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\"\n \"ts\"=\"2026-03-02 09:38:36.174760\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-298\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 09:38:36.616130\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\" \"deployment\"=\"pooledNonMsiScopedAssignments-6h4svl2qyxdco\"\n \"ts\"=\"2026-03-02 09:38:36.835515\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-298\" \"deployment\"=\"msi-deployment-298\"\n \"ts\"=\"2026-03-02 09:38:37.270141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\" \"deployment\"=\"customer-infra-admin-cred-lifecycle-gzjkx6-7znljn\"\n \"ts\"=\"2026-03-02 09:38:38.035630\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-02 09:38:38.035751\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-02 09:38:38.035883\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-lcvrkt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"71c18b1921cd5b17b910b3b72c9066bf643ba475b2448fc0d3518d89149c2324","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"67e9018556a7b506982d3329b29ab5f859a0abeb9d0d95b215a46e7c78893c2d","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b4e8d0\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-dtszn8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-dtszn8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00165e730\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8\",\n },\n \u003c*fmt.wrapErrors | 0xc000b4e810\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-dtszn8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00165e730\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b4e8d0\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-dtszn8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-dtszn8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00165e730\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8\",\n },\n \u003c*fmt.wrapErrors | 0xc000b4e810\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-dtszn8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00165e730\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-dtszn8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"7a446e69e9edc907ba731165f41ca5ec44ba739e49c540c0cdbd035e2a62dbb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"8190aa4cb6a3146f391357b5e008775dfec34a9f3817bc495952f7c9e62c2eae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"429b41ce3ee92b18caddf6a17f863d8668caa5000b4006dd0e0785a3fcfef8d1","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000843890\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gczxqr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f8f340\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr\",\n },\n \u003c*fmt.wrapErrors | 0xc000843860\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gczxqr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f8f340\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000843890\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gczxqr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f8f340\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr\",\n },\n \u003c*fmt.wrapErrors | 0xc000843860\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-gczxqr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f8f340\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"8d9eaf23811aff527f950a97cc99a30b3c6bc4723d53f512bb0003ad81d1901a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"fa7b7d73d51da6e58e7ae409a0fa03992173f6526d5c9cadcfbb3101674486ce","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009768a0\u003e: \n failed to create HCP cluster ea-listdq746l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: failed waiting for cluster=\"ea-listdq746l\" in resourcegroup=\"ea-list-kfdj5p\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listdq746l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: failed waiting for cluster=\\\"ea-listdq746l\\\" in resourcegroup=\\\"ea-list-kfdj5p\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a27630\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p\",\n },\n \u003c*fmt.wrapErrors | 0xc000976870\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listdq746l\\\" in resourcegroup=\\\"ea-list-kfdj5p\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a27630\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009768a0\u003e: \n failed to create HCP cluster ea-listdq746l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: failed waiting for cluster=\"ea-listdq746l\" in resourcegroup=\"ea-list-kfdj5p\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listdq746l, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: failed waiting for cluster=\\\"ea-listdq746l\\\" in resourcegroup=\\\"ea-list-kfdj5p\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a27630\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p\",\n },\n \u003c*fmt.wrapErrors | 0xc000976870\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listdq746l\\\" in resourcegroup=\\\"ea-list-kfdj5p\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a27630\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listdq746l in resource group ea-list-kfdj5p\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"a54555ec616c5b34f8ed6d52567de1f15f87336ef0bca5659ba11787306fb0a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"f4efb522d5be00b347b60743cb47e2a66cff11796f643e659cbad61547616bc9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014a72f0\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-tnnrm7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-tnnrm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00133def0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7\",\n },\n \u003c*fmt.wrapErrors | 0xc0014a72c0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-tnnrm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00133def0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014a72f0\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-tnnrm7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-tnnrm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00133def0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7\",\n },\n \u003c*fmt.wrapErrors | 0xc0014a72c0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-tnnrm7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00133def0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-tnnrm7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"af35176c1e5301669fa1d47c5414f4a6d5b017f6a68fba6c7504308ff8ab3ba9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"f23d526e30484d3f692aa4466b35d93daebc8196cef952714539fc89839c22a7","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016fad50\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-b5xsdx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b5xsdx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014618b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx\",\n },\n \u003c*fmt.wrapErrors | 0xc0016fad20\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b5xsdx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014618b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016fad50\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-b5xsdx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b5xsdx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014618b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx\",\n },\n \u003c*fmt.wrapErrors | 0xc0016fad20\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b5xsdx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014618b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-b5xsdx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"c8df159660cc01f63491ad2720d61af5999dedb312c424a9f0e87077f683c170","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"ef97d45d0044ec40a6299e63e89b378420991d9075a5a3a95e7a9c18aae54d85","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000c64120\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-dn6hbj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-dn6hbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010fe1a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj\",\n },\n \u003c*fmt.wrapErrors | 0xc000c640f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-dn6hbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010fe1a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000c64120\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-dn6hbj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-dn6hbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010fe1a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj\",\n },\n \u003c*fmt.wrapErrors | 0xc000c640f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-dn6hbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010fe1a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dn6hbj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"dced451b5cd309789aa3c2021a14444b28c78c47077542bf4e6060dc9996d3fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"272260ac77068d8e96ef71f76cd00581573aba0d48a5d1409a5994887847fb6f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00108f4a0\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nrtv6d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nrtv6d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f7fb90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d\",\n },\n \u003c*fmt.wrapErrors | 0xc00108f470\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nrtv6d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f7fb90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00108f4a0\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-nrtv6d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nrtv6d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f7fb90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d\",\n },\n \u003c*fmt.wrapErrors | 0xc00108f470\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-nrtv6d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f7fb90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-nrtv6d\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"e7ffd5609d070476b54eb3223101d2f43b174b0e92f46a628e917130510d1abb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"4e5f34a3ab836f1c4c24972165abccacb12fe03b8e8a2b3803ed81a69cef9e79","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000160870\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-r22t5x\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-r22t5x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013018c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x\",\n },\n \u003c*fmt.wrapErrors | 0xc000160840\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-r22t5x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013018c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000160870\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-r22t5x\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-r22t5x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013018c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x\",\n },\n \u003c*fmt.wrapErrors | 0xc000160840\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-r22t5x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013018c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-r22t5x\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"f9d1bf8abcb4da308c96e8d92ea89d7e52c7f9e52c64cdc22672708f8d5f7c68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028367416165142528","occurred_at":"2026-03-02T07:10:51Z","failed_at":"e2e","signature_id":"12ea923a08e606bdbc29877611d0c0e43cba2dedbd69fadf82ad1056805f671f","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d9b60\u003e: \n failed to create HCP cluster list-test-cluster-ldsmp6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: failed waiting for cluster=\"list-test-cluster-ldsmp6\" in resourcegroup=\"cluster-listing-b6c6pj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-ldsmp6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: failed waiting for cluster=\\\"list-test-cluster-ldsmp6\\\" in resourcegroup=\\\"cluster-listing-b6c6pj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001206300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj\",\n },\n \u003c*fmt.wrapErrors | 0xc0014d9b00\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-ldsmp6\\\" in resourcegroup=\\\"cluster-listing-b6c6pj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001206300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d9b60\u003e: \n failed to create HCP cluster list-test-cluster-ldsmp6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: failed waiting for cluster=\"list-test-cluster-ldsmp6\" in resourcegroup=\"cluster-listing-b6c6pj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-ldsmp6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: failed waiting for cluster=\\\"list-test-cluster-ldsmp6\\\" in resourcegroup=\\\"cluster-listing-b6c6pj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001206300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj\",\n },\n \u003c*fmt.wrapErrors | 0xc0014d9b00\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-ldsmp6\\\" in resourcegroup=\\\"cluster-listing-b6c6pj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001206300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ldsmp6 in resource group cluster-listing-b6c6pj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028356367349190656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4266,"pr_state":"closed","pr_sha":"0994ff6436b092928911e70d2576ba0e769a49bf","final_merged_sha":"7fbd7a365361285c704975d46eb93dffa3ec17d6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-02T06:26:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028346787831484416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4266,"pr_state":"closed","pr_sha":"0994ff6436b092928911e70d2576ba0e769a49bf","final_merged_sha":"7fbd7a365361285c704975d46eb93dffa3ec17d6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T05:48:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7e4e257f7094d7a3703c32bde46ac814c99a2bfebc0407c9741ddd25e7362526","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028346787831484416","occurred_at":"2026-03-02T05:48:53Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T05:50:16.906Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T05:50:19.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T05:50:20.887Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d4f70afc8d9311f8f70c291675999760505a5e74ece161170e0d964681b69462\ntime=2026-03-02T05:52:22.007Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1484416/providers/Microsoft.Resources/deployments/d4f70afc8d9311f8f70c291675999760505a5e74ece161170e0d964681b69462/operationStatuses/08584291762655709188\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1484416/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1484416/providers/Microsoft.Monitor/accounts/services-j1484416/providers/Microsoft.Authorization/roleAssignments/f5914e5b-7128-5f36-aa15-b6c046872132\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T055034Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '469f9caa-0aa8-40db-b087-2d96634ba71c', request correlation id '5b201ca7-1bfb-4188-9d0b-08e7d9e0ac7c'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4152/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028327858492936192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4152,"pr_state":"closed","pr_sha":"2f8a0eb8477e5204a65847041c1967c2843f8f80","final_merged_sha":"2f8a0eb8477e5204a65847041c1967c2843f8f80","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-02T04:33:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028322547321278464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4266,"pr_state":"closed","pr_sha":"0994ff6436b092928911e70d2576ba0e769a49bf","final_merged_sha":"7fbd7a365361285c704975d46eb93dffa3ec17d6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T04:12:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"583979653d212e68fffd3d204d39def766aa046112b9423bd5313fa04fd237a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4266/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028322547321278464","occurred_at":"2026-03-02T04:12:34Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T04:52:13.926Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T04:52:16.084Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T04:52:17.247Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a14ce119a510208d0f5a383bc8da5e4080e869520b7a9457f5d30a5dc0b95769\ntime=2026-03-02T04:53:47.971Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1278464/providers/Microsoft.Resources/deployments/a14ce119a510208d0f5a383bc8da5e4080e869520b7a9457f5d30a5dc0b95769/operationStatuses/08584291797488845335\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1278464/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1278464/providers/Microsoft.Monitor/accounts/hcps-j1278464/providers/Microsoft.Authorization/roleAssignments/8cadabfe-91ea-5d67-8477-6254d7bc0076\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T045239Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '2dffd007-f672-41af-aee9-057539f29b1a', request correlation id '1736c96e-5d03-4068-ad2c-2dd1066b35d2'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1278464/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1278464/providers/Microsoft.Monitor/accounts/services-j1278464/providers/Microsoft.Authorization/roleAssignments/17452a88-2648-5301-a931-8abe24121de2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T045240Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '80c572c0-474e-4da8-871d-6abe7bded356', request correlation id '1736c96e-5d03-4068-ad2c-2dd1066b35d2'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4249,"pr_state":"closed","pr_sha":"67bc090ef1f250bde852b94552a8cfc7fe13e836","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T02:51:46Z"},"failed_at":["e2e"],"failed_test_count":18,"occurrences":[{"row_id":"1e7b5d5b9dbb63cf21334d5f6dca79f0f9211ad979b522849e82555c3defe60f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"5b4be603a66d063d837c81da1af96838339775aa7ea4f141df35cbcb46dc9632","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a7cd80\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-f56j4k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-f56j4k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0019a0dc0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k\",\n },\n \u003c*fmt.wrapErrors | 0xc000a7cd50\u003e{\n msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-f56j4k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0019a0dc0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a7cd80\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-f56j4k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-f56j4k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0019a0dc0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k\",\n },\n \u003c*fmt.wrapErrors | 0xc000a7cd50\u003e{\n msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-f56j4k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0019a0dc0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-f56j4k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"2363ae0e890c775a37cb64a3cbc443ac6aa836865f58bf5a5c2725e41e47f90a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"a834318e05bb8778ddfb3fcaedfbc9a6b9aeec18b655b5b58bce1a7f4570a6c8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f58ff0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-h8xwhm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-h8xwhm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ffc0a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm\",\n },\n \u003c*fmt.wrapErrors | 0xc000f58fc0\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-h8xwhm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ffc0a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f58ff0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-h8xwhm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-h8xwhm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ffc0a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm\",\n },\n \u003c*fmt.wrapErrors | 0xc000f58fc0\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-h8xwhm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ffc0a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-h8xwhm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"23d9b64ea22d592d428cc03c49d03919ad8fafd18b08e16d84c9c435683da909","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"6a2a55de8c481a7ce380a5e9056e343ca779b8848a783449054afb8c67231b6b","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7eff59016260, 0xc00078d950})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00078d950?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 95\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/02/26 05:15:59.872\n \"ts\"=\"2026-03-02 05:15:59.872456\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-02 05:15:59.872532\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6gjd8m\" \"resourceGroup\"=\"customer-rg-46vn7h\"\n \"ts\"=\"2026-03-02 05:16:00.486714\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-6gjd8m\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 05:16:01.506413\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-492sc8\" \"resourceGroup\"=\"customer-rg-46vn7h\"\n \"ts\"=\"2026-03-02 05:16:01.667950\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-492sc8\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 05:16:02.425889\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 05:16:02.425981\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 05:16:02.426036\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-02 05:16:02.427057\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-280\"\n \"ts\"=\"2026-03-02 05:16:02.427097\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-244\"\n \"ts\"=\"2026-03-02 05:16:02.427139\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-189\"\n \"ts\"=\"2026-03-02 05:16:02.427162\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-46vn7h\"\n \"ts\"=\"2026-03-02 05:16:03.129001\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-280\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 05:16:03.245854\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-244\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 05:16:03.680660\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-189\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 05:16:04.046614\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-02 05:16:04.480114\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-280\" \"deployment\"=\"msi-deployment-280\"\n \"ts\"=\"2026-03-02 05:16:04.723671\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-244\" \"deployment\"=\"msi-deployment-244\"\n \"ts\"=\"2026-03-02 05:16:05.177706\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-189\" \"deployment\"=\"msi-deployment-189\"\n \"ts\"=\"2026-03-02 05:16:05.432294\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-492sc8\"\n \"ts\"=\"2026-03-02 05:16:06.839363\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-02 05:16:07.282485\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6gjd8m\"\n \"ts\"=\"2026-03-02 05:16:07.741013\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-02 05:16:07.741142\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-02 05:16:07.741278\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-46vn7h\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7eff59016260, 0xc00078d950})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00078d950?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 95\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/02/26 05:15:59.872\n \"ts\"=\"2026-03-02 05:15:59.872456\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-02 05:15:59.872532\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6gjd8m\" \"resourceGroup\"=\"customer-rg-46vn7h\"\n \"ts\"=\"2026-03-02 05:16:00.486714\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-6gjd8m\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 05:16:01.506413\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-492sc8\" \"resourceGroup\"=\"customer-rg-46vn7h\"\n \"ts\"=\"2026-03-02 05:16:01.667950\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-492sc8\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-02 05:16:02.425889\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 05:16:02.425981\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-02 05:16:02.426036\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-02 05:16:02.427057\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-280\"\n \"ts\"=\"2026-03-02 05:16:02.427097\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-244\"\n \"ts\"=\"2026-03-02 05:16:02.427139\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-189\"\n \"ts\"=\"2026-03-02 05:16:02.427162\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-46vn7h\"\n \"ts\"=\"2026-03-02 05:16:03.129001\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-280\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 05:16:03.245854\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-244\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 05:16:03.680660\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-189\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-02 05:16:04.046614\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-02 05:16:04.480114\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-280\" \"deployment\"=\"msi-deployment-280\"\n \"ts\"=\"2026-03-02 05:16:04.723671\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-244\" \"deployment\"=\"msi-deployment-244\"\n \"ts\"=\"2026-03-02 05:16:05.177706\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-189\" \"deployment\"=\"msi-deployment-189\"\n \"ts\"=\"2026-03-02 05:16:05.432294\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-492sc8\"\n \"ts\"=\"2026-03-02 05:16:06.839363\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-02 05:16:07.282485\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-46vn7h\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6gjd8m\"\n \"ts\"=\"2026-03-02 05:16:07.741013\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-02 05:16:07.741142\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-02 05:16:07.741278\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-46vn7h\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"3bf017ac7e6e8afa87a321f7a562c9cf9490b0f19532da335cf172fbcf0517e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"0ce16d416659a5f3b49779cf651dd87614171893c8679f452b5f2c3bcbe91167","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000bd0ea0\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tf9sbl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-tf9sbl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009ebf40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl\",\n },\n \u003c*fmt.wrapErrors | 0xc000bd0e70\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-tf9sbl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009ebf40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000bd0ea0\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tf9sbl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-tf9sbl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009ebf40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl\",\n },\n \u003c*fmt.wrapErrors | 0xc000bd0e70\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-tf9sbl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009ebf40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-tf9sbl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"449ba9cd9e6ff3d740356c407ff216904730628b60b4941ed1bfcb17c7c060ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"20bd7eb0a21eb202db73f55b9238c0c05ce37712aaeb5215115eca38a0b98f19","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007f5050\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-gxntjr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-gxntjr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000de1980\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr\",\n },\n \u003c*fmt.wrapErrors | 0xc0007f5020\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-gxntjr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000de1980\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007f5050\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-gxntjr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-gxntjr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000de1980\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr\",\n },\n \u003c*fmt.wrapErrors | 0xc0007f5020\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-gxntjr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000de1980\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gxntjr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"62585b1ac1743e8ed06ac247cf64a9ba5c63146bc587b121a610b30c00e6b89a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"25311bd66d07be4c48e0263bffdf849edee50925961dddba206e0c2286a1af8c","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00143b2c0\u003e: \n failed to create HCP cluster list-test-cluster-hrf7pn, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: failed waiting for cluster=\"list-test-cluster-hrf7pn\" in resourcegroup=\"cluster-listing-hjgfcm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-hrf7pn, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: failed waiting for cluster=\\\"list-test-cluster-hrf7pn\\\" in resourcegroup=\\\"cluster-listing-hjgfcm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e57ab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm\",\n },\n \u003c*fmt.wrapErrors | 0xc00143b290\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-hrf7pn\\\" in resourcegroup=\\\"cluster-listing-hjgfcm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e57ab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00143b2c0\u003e: \n failed to create HCP cluster list-test-cluster-hrf7pn, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: failed waiting for cluster=\"list-test-cluster-hrf7pn\" in resourcegroup=\"cluster-listing-hjgfcm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster list-test-cluster-hrf7pn, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: failed waiting for cluster=\\\"list-test-cluster-hrf7pn\\\" in resourcegroup=\\\"cluster-listing-hjgfcm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e57ab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm\",\n },\n \u003c*fmt.wrapErrors | 0xc00143b290\u003e{\n msg: \"failed waiting for cluster=\\\"list-test-cluster-hrf7pn\\\" in resourcegroup=\\\"cluster-listing-hjgfcm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e57ab0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-hrf7pn in resource group cluster-listing-hjgfcm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"741f46b40cfd651499096aa0e958f335eac55c0c1098be439e57fca6f3d384c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"d755359d8261f060b13d7a551860e237562abad6aea96358a1608800bb00142a","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0004fe5a0\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-2dg8kb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-2dg8kb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c0a2a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb\",\n },\n \u003c*fmt.wrapErrors | 0xc0004fe570\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-2dg8kb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c0a2a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0004fe5a0\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-2dg8kb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-2dg8kb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c0a2a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb\",\n },\n \u003c*fmt.wrapErrors | 0xc0004fe570\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-2dg8kb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c0a2a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-2dg8kb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"7d422e0b16a380d0c2465ad194e5cc817b903adb28c3d7b21fb596bd8bf1e1ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"6d13a1778fd9cbf4421e16744d01d25887007964c1a34b173b3bb14b5e2fdd92","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000848000\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-hwtv2t\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hwtv2t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001605100\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t\",\n },\n \u003c*fmt.wrapErrors | 0xc000d67bc0\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hwtv2t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001605100\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000848000\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-hwtv2t\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hwtv2t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001605100\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t\",\n },\n \u003c*fmt.wrapErrors | 0xc000d67bc0\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-hwtv2t\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001605100\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-hwtv2t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"819bac517b60ee6b9e243148139c2156e3d49b84df8d32e390f08cc51843c397","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"4fa80c38504d5d77773dbad78afa704f083cb85f1ffae914d3ddfb381b28f4b6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014c68a0\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gxmnpw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gxmnpw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001083650\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw\",\n },\n \u003c*fmt.wrapErrors | 0xc0014c6870\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gxmnpw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001083650\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014c68a0\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gxmnpw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gxmnpw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001083650\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw\",\n },\n \u003c*fmt.wrapErrors | 0xc0014c6870\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-gxmnpw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001083650\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-gxmnpw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"88428e61b4ad6130119af290b0b62a785fd6c726b4e660771e765694b020f5d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"a11fecdc6f9c9f502c84b213a2c6ddcb1312a7fb20a346402a230f9a22dd6027","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001510e10\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2gq4c5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2gq4c5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c5960\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5\",\n },\n \u003c*fmt.wrapErrors | 0xc001510de0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2gq4c5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c5960\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001510e10\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2gq4c5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2gq4c5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c5960\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5\",\n },\n \u003c*fmt.wrapErrors | 0xc001510de0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2gq4c5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c5960\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-2gq4c5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"91e1c8ca5cf7f2ffd918f2cfb2a836195a863b5483344fb65d9a659fd1339913","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"7dc26f820fd2928c111033bd0ce08af1bae6042a6cbe5fb5bf76d461b5be1d01","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"9deab460673c91c72aac1f8b937e71416a548e3cccbf08072dad5c1bb01f7d04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"5d8313015f232a71fc8cce70daa74e925e3a061aeb79f6380b76d3d4b524e046","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016365a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-x2jwxn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-x2jwxn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012e6740\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn\",\n },\n \u003c*fmt.wrapErrors | 0xc001636570\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-x2jwxn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012e6740\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016365a0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-x2jwxn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-x2jwxn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012e6740\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn\",\n },\n \u003c*fmt.wrapErrors | 0xc001636570\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-x2jwxn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012e6740\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-x2jwxn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"b78ca069c5ea563deeb18b9a7ba17646a81e9cdb3551b2bbb481071267bb73bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"79e0fa75d21d21b037b5bb751aa81505c43ec44bc254abe1d1d58608dd4cd83b","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008b5ad0\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-d42j55\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-d42j55\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010c99b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55\",\n },\n \u003c*fmt.wrapErrors | 0xc0008b5aa0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-d42j55\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010c99b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008b5ad0\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-d42j55\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-d42j55\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010c99b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55\",\n },\n \u003c*fmt.wrapErrors | 0xc0008b5aa0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-d42j55\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010c99b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-d42j55\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"bc52a1b9427e60d412f7c8ff2bee646af23a429057c7878cde72138bc17ce8fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"e7b004e53d8832013a3d194a72c863daf5c51b2c65dccdd432809407b71932f2","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015d4d80\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-xsr2z8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xsr2z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001756360\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8\",\n },\n \u003c*fmt.wrapErrors | 0xc0015d4d50\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xsr2z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001756360\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015d4d80\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-xsr2z8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xsr2z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001756360\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8\",\n },\n \u003c*fmt.wrapErrors | 0xc0015d4d50\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-xsr2z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001756360\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-xsr2z8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"dee1ec108bd7f8037d1ed078b5552cd6886c8325c7a7c91a5a5232d69c2565ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"2bf5699ff122f25bc565b58dd3853609f3e854c72d8b9723bb006ad377aa6759","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00140d530\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-psx5nr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-psx5nr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bed50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr\",\n },\n \u003c*fmt.wrapErrors | 0xc00140d4d0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-psx5nr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bed50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00140d530\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-psx5nr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-psx5nr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bed50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr\",\n },\n \u003c*fmt.wrapErrors | 0xc00140d4d0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-psx5nr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bed50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-psx5nr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"e39157216332b45e8c0f41042594140377ce1320b01c8c644f5b8918fe9f03d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"f1c4d87f28d7c5539ad3798af0212fa5687227706ff59d66fecc74dc585e34ef","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001606a20\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-szlg27\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-szlg27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00155e530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27\",\n },\n \u003c*fmt.wrapErrors | 0xc0016069f0\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-szlg27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00155e530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001606a20\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-szlg27\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-szlg27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00155e530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27\",\n },\n \u003c*fmt.wrapErrors | 0xc0016069f0\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-szlg27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00155e530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-szlg27\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"eec6d8c50b374be11db7c5623237503022aa3f0acc145eec92f9476320a7ac08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"9be0574edc48543bc89400eb6240cb9e16f02184d4d19361133d408a15aee82e","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00120c2a0\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-pt7ssl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pt7ssl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012979c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl\",\n },\n \u003c*fmt.wrapErrors | 0xc00120c270\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pt7ssl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012979c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00120c2a0\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-pt7ssl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pt7ssl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012979c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl\",\n },\n \u003c*fmt.wrapErrors | 0xc00120c270\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-pt7ssl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012979c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-pt7ssl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}},{"row_id":"f6009d86aa71b7d40d42a66185a3320fcebdaacac8d0f943c8926e989c7814a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028302213423566848","occurred_at":"2026-03-02T02:51:46Z","failed_at":"e2e","signature_id":"df3b43f96bbd2a7a9e928e7b64e75a06f5879e7fabc317f7c1a0adcce9302925","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00069adb0\u003e: \n failed to create HCP cluster ea-list448xs9, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: failed waiting for cluster=\"ea-list448xs9\" in resourcegroup=\"ea-list-lq9mft\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-list448xs9, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: failed waiting for cluster=\\\"ea-list448xs9\\\" in resourcegroup=\\\"ea-list-lq9mft\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000eeb1f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft\",\n },\n \u003c*fmt.wrapErrors | 0xc00069ad80\u003e{\n msg: \"failed waiting for cluster=\\\"ea-list448xs9\\\" in resourcegroup=\\\"ea-list-lq9mft\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000eeb1f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00069adb0\u003e: \n failed to create HCP cluster ea-list448xs9, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: failed waiting for cluster=\"ea-list448xs9\" in resourcegroup=\"ea-list-lq9mft\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-list448xs9, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: failed waiting for cluster=\\\"ea-list448xs9\\\" in resourcegroup=\\\"ea-list-lq9mft\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000eeb1f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft\",\n },\n \u003c*fmt.wrapErrors | 0xc00069ad80\u003e{\n msg: \"failed waiting for cluster=\\\"ea-list448xs9\\\" in resourcegroup=\\\"ea-list-lq9mft\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000eeb1f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list448xs9 in resource group ea-list-lq9mft\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-gczxqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-gczxqr, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028290818267156480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-02T02:06:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4808aaec691cba8160da909929dd86cf2ac589293c07c41b1494a94cf639d597","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2028290818267156480","occurred_at":"2026-03-02T02:06:29Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-02T02:37:16.440Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-02T02:37:19.014Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-02T02:37:20.305Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e2e0cdaa8d06319be6584a2f2021fa7f2c7d9a5015775635bd70c205792d0573\ntime=2026-03-02T02:38:50.988Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7156480/providers/Microsoft.Resources/deployments/e2e0cdaa8d06319be6584a2f2021fa7f2c7d9a5015775635bd70c205792d0573/operationStatuses/08584291878459340216\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7156480/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7156480/providers/Microsoft.Monitor/accounts/hcps-j7156480/providers/Microsoft.Authorization/roleAssignments/d65727bf-e0db-5c0e-af88-21e98a9a581a\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260302T023742Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9b9d9c8d-38bd-4b83-b33b-55ee658f0de5', request correlation id 'ee4d00aa-ba4a-41a1-8022-99c4133f274a'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}