{"meta":{"date":"2026-03-27","timezone":"UTC","generated_at":"2026-10-31T23:15:19Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":29,"failed_runs":14,"runs_with_occurrences":14,"runs_with_failure_pattern_matches":11,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":15,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4664/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037676734576332800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4664,"pr_state":"closed","pr_sha":"a319b384ee60f5e4d6bd446b908fd1bb0dfd5648","final_merged_sha":"a319b384ee60f5e4d6bd446b908fd1bb0dfd5648","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-27T23:42:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037672056534863872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4638,"pr_state":"closed","pr_sha":"2ffae64cafef08b4c7e473a51483818581bcee76","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T23:24:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037603036829847552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"b7c1602e423bd01170d8321437755742c0f63c7c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T18:49:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"896b40581caf74b8d49c8b298a0085ae695bc01650d9003a9b581266d3f0afba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037603036829847552","occurred_at":"2026-03-27T18:49:55Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-03-27T19:19:05.051Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-03-27T19:19:05.846Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-03-27T19:19:07.381Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=ebfac1080e8056bda95a598be1328338af0d62013965b54fd55fc53065574571\ntime=2026-03-27T19:31:10.214Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deployments/ebfac1080e8056bda95a598be1328338af0d62013965b54fd55fc53065574571/operationStatuses/08584269677389986705\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deployments/maestro-consumer\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deployments/maestro-eg-crt-kmvtx2yprmvuq\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deployments/maestro-eg-crt-kmvtx2yprmvuq\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deployments/hcp-underlay-j9847552-mgmt-1-cert\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deployments/hcp-underlay-j9847552-mgmt-1-cert\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9847552-mgmt-1/providers/Microsoft.Resources/deploymentScripts/newCert-hcp-underlay-j9847552-mgmt-1\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\\\\\",\\\\r\\\\n                            \\\\\\\"details\\\\\\\": [\\\\r\\\\n                              {\\\\r\\\\n                                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentScriptOperationFailed\\\\\\\",\\\\r\\\\n                                \\\\\\\"message\\\\\\\": \\\\\\\"A task was canceled.\\\\\\\"\\\\r\\\\n                              }\\\\r\\\\n                            ]\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e816f1e3b47b563efe8a6cd99a62f5abdf0ac3394314f730c55c2f66121c5a89","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentScriptOperationFailed; detail message A task was canceled.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e816f1e3b47b563efe8a6cd99a62f5abdf0ac3394314f730c55c2f66121c5a89"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4610,"pr_state":"open","pr_sha":"9e0509813fc4c1275f88ebcefdb3efb502fd9a76","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T18:37:06Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"062caca0a65de7e93ffeaf46b98420b1e06af52c733dd21c0e5601c83f3917a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"174aae22b043c787f4ff01904cf52fb63d8a6c6d43576cd36eddcb790e831615","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"ca544c7094196727b35410889b12fcb40986d919048fe43815d8dfa30f5bdac8","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:234]: Unexpected error:\n    \u003c*errors.joinError | 0xc001180600\u003e: \n    invalid node pool version update: expected error but none occurred\n    updating immutable fields: expected error containing keywords 'forbidden field immutable', got: PUT https://rp.j2393088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*errors.errorString | 0xc000955650\u003e{\n                s: \"invalid node pool version update: expected error but none occurred\",\n            },\n            \u003c*errors.errorString | 0xc000e49960\u003e{\n                s: \"updating immutable fields: expected error containing keywords 'forbidden field immutable', got: PUT https://rp.j2393088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:234]: Unexpected error:\n    \u003c*errors.joinError | 0xc001180600\u003e: \n    invalid node pool version update: expected error but none occurred\n    updating immutable fields: expected error containing keywords 'forbidden field immutable', got: PUT https://rp.j2393088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*errors.errorString | 0xc000955650\u003e{\n                s: \"invalid node pool version update: expected error but none occurred\",\n            },\n            \u003c*errors.errorString | 0xc000e49960\u003e{\n                s: \"updating immutable fields: expected error containing keywords 'forbidden field immutable', got: PUT https://rp.j2393088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-7wccc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1019af4c9c47d1aa706702f78998f006a0e49b021ad920d44d356df2637fa18e","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"1e715b9623bdfeab6fb3a21bb55d839eb73c9bf4a4efb0f7e4dd239769192f76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"57e33998a513a84770bf54156435241e91c0dfbc020daeaf86ebc8a25e3d452e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:255]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d60f60\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zmsvbg\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zmsvbg\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009b4060\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:255]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d60f60\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zmsvbg\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zmsvbg\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009b4060\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d75e828705d5e967c897d0839b4327d675cfc294e87dead6ef776571e28083d","failure_pattern":"UpdateHCPCluster20251223 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zmsvbg\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-zmsvbg, error: context deadline exceeded"}},{"row_id":"273925049e653622676df8b5df0146488152ceb6a3a65bc77554eecd5dccf420","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"45b3b973aa4d9d28512dc327a56e951b3132227b1ebad9ef7bef8ddee29d10d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"4b8e4229f32f38411ec6799c8f47f1eda7edcfd1dad82cbc9b4af6807006f090","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bc2e10\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hfjx5x\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hfjx5x, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-hfjx5x\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hfjx5x, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00020cac0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hfjx5x\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bc2e10\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hfjx5x\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hfjx5x, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-hfjx5x\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hfjx5x, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00020cac0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hfjx5x\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-kk24qw\" in resourcegroup=\"admin-credential-lifecycle-test-n47zt9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded"}},{"row_id":"4e1e3395d2a40ed48d0708312efd390320f99f073b31532104771c870773de6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"47c2e3264eae43e915bbed7d7f54d5b0b9955e1b2d3bbe46c8034998baf5a071","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001208168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-b2nl5w\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000cf1100\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-b2nl5w\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000cf10e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-b2nl5w\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000cf10c0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-b2nl5w\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b17d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001208168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-b2nl5w\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000cf1100\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-b2nl5w\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000cf10e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-b2nl5w\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000cf10c0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-b2nl5w\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b17d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-b2nl5w\" to finish deleting: context canceled"}},{"row_id":"6e117ced86891c60d0484fdaafda0d4662b9e285e269b14a0264eefc737e1461","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"279ffa68f7f1689a00809ef14b6bf5a58b34426b2e120e4732ef6cece31c2ffc","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a293e0\u003e: \n    failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4h255n\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-4h255n\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000095810\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a293e0\u003e: \n    failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4h255n\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-4h255n\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000095810\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4h255n\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4h255n, error: context deadline exceeded"}},{"row_id":"92dce4cce17b2ea54fd2584fc9072ae2aa71094927fbe6a5d5f1d91fe405e617","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"844abb626ee68aa13da5189e5097bfa6ff9073bdb332c2b6854f120e3a48b656","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e4090\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-vjd65z\" in resourcegroup=\"rg-zstream-upgrade-4-21-vbxrjb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-vjd65z in resource group rg-zstream-upgrade-4-21-vbxrjb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-vjd65z\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-vbxrjb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-vjd65z in resource group rg-zstream-upgrade-4-21-vbxrjb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a802d0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-vjd65z in resource group rg-zstream-upgrade-4-21-vbxrjb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e4090\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-vjd65z\" in resourcegroup=\"rg-zstream-upgrade-4-21-vbxrjb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-vjd65z in resource group rg-zstream-upgrade-4-21-vbxrjb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-vjd65z\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-vbxrjb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-vjd65z in resource group rg-zstream-upgrade-4-21-vbxrjb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a802d0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-vjd65z in resource group rg-zstream-upgrade-4-21-vbxrjb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-kk24qw\" in resourcegroup=\"admin-credential-lifecycle-test-n47zt9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded"}},{"row_id":"afade1a8284f119cc01c2eca764bc54c3f129db88b0488d7b5044fa610f25cf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"b9f927250425b24f8fb8ef9923c50771f49fec8b1b899c50a40ed3f4e41328be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"f202f531c5c7e55ca8663d6ebd1f373e6392032f901f1c967d2641d0c95d1aac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037599811762393088","occurred_at":"2026-03-27T18:37:06Z","failed_at":"e2e","signature_id":"5f345a29bfb7ccb08311c39116dac676d6cd25aaf3728b1779dbd5669b9829ea","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0006377d0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-kk24qw\" in resourcegroup=\"admin-credential-lifecycle-test-n47zt9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kk24qw\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-n47zt9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0007a6c10\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0006377d0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-kk24qw\" in resourcegroup=\"admin-credential-lifecycle-test-n47zt9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kk24qw\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-n47zt9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0007a6c10\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-kk24qw\" in resourcegroup=\"admin-credential-lifecycle-test-n47zt9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kk24qw in resource group admin-credential-lifecycle-test-n47zt9, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["0d75e828705d5e967c897d0839b4327d675cfc294e87dead6ef776571e28083d","1019af4c9c47d1aa706702f78998f006a0e49b021ad920d44d356df2637fa18e","15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037570874772557824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4663,"pr_state":"closed","pr_sha":"ff8a7a7ab7c8a2404988d0e63c0784f22a0b2e96","final_merged_sha":"ff8a7a7ab7c8a2404988d0e63c0784f22a0b2e96","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-27T16:41:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037570752298881024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4663,"pr_state":"closed","pr_sha":"e55f8cf98aa1f2c3ff67cdadcc39d4f0b425166c","final_merged_sha":"ff8a7a7ab7c8a2404988d0e63c0784f22a0b2e96","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T16:41:27Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"38036915d0bdb76cf8e6866eba6201fd8614ce7374a3ae1cab0f28eabd595b3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037570752298881024","occurred_at":"2026-03-27T16:41:27Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037570472102596608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4663,"pr_state":"closed","pr_sha":"84e3c8b1a0cda61e9e90676e7c52fe086381a792","final_merged_sha":"ff8a7a7ab7c8a2404988d0e63c0784f22a0b2e96","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T16:39:48Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b140c5f4c08060e61cb4d220457e8af9320ab8c07d86079c6bf8c70e861fbb05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037570472102596608","occurred_at":"2026-03-27T16:39:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037565455471742976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"73e9932cf6489e5454cd45ab88bc2a7aa523d290","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T16:20:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c9dc07da215d0771adc617d8b58f43cac4b63935c79ab6317a4cd67720043b84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037565455471742976","occurred_at":"2026-03-27T16:20:35Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-27T16:55:49.963Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-27T16:55:51.467Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-27T16:55:53.166Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=030e0412555f7d04adbd61e54aa5cefd5e9e1d5fd8b3b060b5ee4a73f2db320a\ntime=2026-03-27T17:08:56.231Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1742976-svc/providers/Microsoft.Resources/deployments/030e0412555f7d04adbd61e54aa5cefd5e9e1d5fd8b3b060b5ee4a73f2db320a/operationStatuses/08584269763333001275\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1742976-svc/providers/Microsoft.Resources/deployments/cluster-b4ajflcf262bs\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1742976-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1742976-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1742976-svc/providers/Microsoft.ContainerService/managedClusters/prow-j1742976-svc/agentPools/ud8dsv62\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"original error: Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1742976-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-ud8dsv62-21047250-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n. Please note that allocation failures can also arise if a proximity placement group is associated with this VMSS.  See https://learn.microsoft.com/en-us/troubleshoot/azure/azure-kubernetes/error-code-zonalallocationfailed-allocationfailed for more details.  This is not AKS controlled behavior, please ask help to VMSS team for allocation failure.  If Node Auto Provisioner (NAP) is not enabled,  consider enabling NAP to designate multiple SKUs for higher chance of success, see https://aka.ms/aks/enableNAP.  If NAP is already enabled,  consider adding additional SKUs to the NAP configuration for higher chance of success, see https://aka.ms/aks/configNAP.\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037560930929217536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4662,"pr_state":"closed","pr_sha":"18197a99198177e92e1444a07e067ca0fcb3ac05","final_merged_sha":"7b198365c9a7d376c29313527fcad41caa96db77","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T16:02:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037555177531117568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4661,"pr_state":"closed","pr_sha":"e04590ea5be7ae7e35c6aba3fd96b18118617ef4","final_merged_sha":"6b40798b67d3b3505633135ea9bb036a6ce6f947","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T15:39:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037541962378121216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"8e24d4897749896bf0f948a65b57805749e48f80","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T14:47:09Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"75d344511c7dc4086fa73d0873bca07041092e7eb0c180b56a3d74e0e5b9739e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037541962378121216","occurred_at":"2026-03-27T14:47:09Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-27T15:06:36.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-27T15:06:38.469Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-27T15:06:39.655Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6b04bb802b99afb35d3a7da01b435a2fe54542974d8c55f4de7c0fd1253cb1ef\ntime=2026-03-27T15:12:11.166Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8121216/providers/Microsoft.Resources/deployments/6b04bb802b99afb35d3a7da01b435a2fe54542974d8c55f4de7c0fd1253cb1ef/operationStatuses/08584269828865178334\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8121216/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8121216/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j8121216\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report cdabde50-a0ae-4404-900f-94278705eafb:3/27/2026 3:09:17 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037538517214564352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"d377d9d2de8a4d7611f6f6890c999df8dd8ee427","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T14:33:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037525260554735616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4607,"pr_state":"closed","pr_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","final_merged_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-27T13:38:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037522866286628864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4659,"pr_state":"closed","pr_sha":"0ab65ffb37ef2dc86396464f9ff2e4bba799b018","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T13:28:07Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"352edc1888a3c8936996404f5565a3ae932f8a6a8f190737705b620dde6c741d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037522866286628864","occurred_at":"2026-03-27T13:28:07Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-27T13:50:41.577Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-27T13:50:43.752Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-27T13:50:45.176Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6be2ba751dc97e52c6d9bfdb5b511a47d6811cd308995a068a3f14585b18f93b\ntime=2026-03-27T13:53:46.117Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6628864/providers/Microsoft.Resources/deployments/6be2ba751dc97e52c6d9bfdb5b511a47d6811cd308995a068a3f14585b18f93b/operationStatuses/08584269874412386907\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6628864/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6628864/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j6628864\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 39fe6ca5-272a-4487-8e34-a1281db54110:3/27/2026 1:52:06 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4625/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037518439836291072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4625,"pr_state":"closed","pr_sha":"7538049ed0af65a70174051daaac76d802496717","final_merged_sha":"d00be7a802af61e780e9a2815cffe5e799eba275","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T13:13:45Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3b959afcdaee41b7d8e4b0cce17b0a26a28bbfb108bf10652a7e70ab57af3d6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4625/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037518439836291072","occurred_at":"2026-03-27T13:13:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037506083576090624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4659,"pr_state":"closed","pr_sha":"0ab65ffb37ef2dc86396464f9ff2e4bba799b018","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T12:24:36Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"18f9cfa33c7f2c28b29f026a99ad3a1ef0c3557639c663b8b203fa988ed27858","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037506083576090624","occurred_at":"2026-03-27T12:24:36Z","failed_at":"provision","signature_id":"4f1834b4479995557b5a77691464b0b9ec70391551ba8b11ff826a2f2fb6878d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp","test_suite":"step graph","failure_text":"time=2026-03-27T13:07:17.428Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-03-27T13:07:17.868Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp\ntime=2026-03-27T13:07:18.596Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp deployment=d16a041e49e06e1f6d817ab604587a607675dab554c68cbf6dfe492508b4735f\ntime=2026-03-27T13:08:48.986Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6090624-mgmt-1/providers/Microsoft.Resources/deployments/d16a041e49e06e1f6d817ab604587a607675dab554c68cbf6dfe492508b4735f/operationStatuses/08584269900473610651\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6090624-mgmt-1/providers/Microsoft.Resources/deployments/profile-expinhr3wegwq\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6090624-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j6090624-mgmt-1/resourceAssociations/nsp-j6090624-mgmt-1-ra-5f8c9afe-2b58-50af-bcb9-ea31ab3d5179\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"ServiceUnavailable\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6090624-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j6090624-mgmt-1/resourceAssociations/nsp-j6090624-mgmt-1-ra-5f8c9afe-2b58-50af-bcb9-ea31ab3d5179, Error message: The service is unavailable now. Please retry the request later.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ff9a8c2e19a12fa1829a0933fd184e474ef9d2c407dcb0e75f3a65d498dc938","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ServiceUnavailable; detail message Error while processing network security perimeter association: The service is unavailable now. Please retry the request later.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ff9a8c2e19a12fa1829a0933fd184e474ef9d2c407dcb0e75f3a65d498dc938"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037498572793974784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4659,"pr_state":"closed","pr_sha":"0ab65ffb37ef2dc86396464f9ff2e4bba799b018","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T11:54:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e78589687689eb40c33e951513e3207e1e8b3e46e0adeaf0a5dd0acf5a97ed8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037498572793974784","occurred_at":"2026-03-27T11:54:49Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-27T12:12:30.979Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-27T12:12:33.010Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-27T12:12:34.380Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5ad762b00291321f96cc50f58624231bd6cdcebe8741580f3f82f1cf16401693\ntime=2026-03-27T12:15:05.312Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j3974784/providers/Microsoft.Resources/deployments/5ad762b00291321f96cc50f58624231bd6cdcebe8741580f3f82f1cf16401693/operationStatuses/08584269933320118051\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3974784/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3974784/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j3974784\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9f26347b-934c-4bcd-a70a-9dbc2b6b0253:3/27/2026 12:14:54 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037484105989361664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4659,"pr_state":"closed","pr_sha":"0ab65ffb37ef2dc86396464f9ff2e4bba799b018","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T10:57:20Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5812948871cf3f4f875fcea8ebc4adadc6caa8a3477fdef234308bd484b04c76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037484105989361664","occurred_at":"2026-03-27T10:57:20Z","failed_at":"provision","signature_id":"4f1834b4479995557b5a77691464b0b9ec70391551ba8b11ff826a2f2fb6878d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp","test_suite":"step graph","failure_text":"time=2026-03-27T11:34:10.047Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-03-27T11:34:10.620Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp\ntime=2026-03-27T11:34:11.320Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp deployment=7b3e71b5bd4dbed737a5f0fd102127d346b93c4808ef9226ab4edba648c35d06\ntime=2026-03-27T11:35:11.625Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9361664-mgmt-1/providers/Microsoft.Resources/deployments/7b3e71b5bd4dbed737a5f0fd102127d346b93c4808ef9226ab4edba648c35d06/operationStatuses/08584269956345969368\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9361664-mgmt-1/providers/Microsoft.Resources/deployments/profile-duhqqw2zn5gxa\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9361664-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9361664-mgmt-1/resourceAssociations/nsp-j9361664-mgmt-1-ra-f67c4493-c44d-5369-ad4a-1db781032435\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"ServiceUnavailable\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9361664-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j9361664-mgmt-1/resourceAssociations/nsp-j9361664-mgmt-1-ra-f67c4493-c44d-5369-ad4a-1db781032435, Error message: The service is unavailable now. Please retry the request later.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ff9a8c2e19a12fa1829a0933fd184e474ef9d2c407dcb0e75f3a65d498dc938","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ServiceUnavailable; detail message Error while processing network security perimeter association: The service is unavailable now. Please retry the request later.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ff9a8c2e19a12fa1829a0933fd184e474ef9d2c407dcb0e75f3a65d498dc938"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037480793080598528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4641,"pr_state":"closed","pr_sha":"6a659af9d2b69b8b76999874fc48a3353e46370b","final_merged_sha":"6a659af9d2b69b8b76999874fc48a3353e46370b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-27T10:44:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037475880200048640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4657,"pr_state":"closed","pr_sha":"b2ad126238ff2b6f338f9dbf6c418bd04799aad8","final_merged_sha":"b2ad126238ff2b6f338f9dbf6c418bd04799aad8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-27T10:24:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037456292993306624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4658,"pr_state":"closed","pr_sha":"5a497f33d94504ce03fbaa221c330d5a9b2f1a55","final_merged_sha":"e1d971e3bab37351e749c3397ce24152d342cd8a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T09:06:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037449664973770752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4645,"pr_state":"closed","pr_sha":"3c268db170e9e06594d21b2304649098bf1d1798","final_merged_sha":"c66c503c1b9d5df065741762f0079c31491b75e8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T08:40:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037446537365164032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4657,"pr_state":"closed","pr_sha":"c79394e578432df901bfbbf776b14e2c037bca39","final_merged_sha":"b2ad126238ff2b6f338f9dbf6c418bd04799aad8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T08:28:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"82233e624e67a4907beb47df2f214634e747eb391a6a62f7b58cca7d3c4dd6cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037446537365164032","occurred_at":"2026-03-27T08:28:03Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-27T08:56:40.881Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-27T08:56:42.307Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-27T08:56:44.504Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=c37be19bb954b8b72ce6236bb195a49bfafdd97e7dbf2d5eb7f7756455bb09ac\ntime=2026-03-27T09:08:17.636Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.Resources/deployments/c37be19bb954b8b72ce6236bb195a49bfafdd97e7dbf2d5eb7f7756455bb09ac/operationStatuses/08584270050820657013\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.Resources/deployments/cluster-pnhfgkbajis54\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.ContainerService/managedClusters/prow-j5164032-svc/agentPools/usd8sv62\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"VMSizeDoesNotSupportEphemeralOS\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"The Virtual Machine size Standard_D8s_v6 does not support Ephemeral OS disk.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.ContainerService/managedClusters/prow-j5164032-svc/agentPools/usd8sv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"VMSizeDoesNotSupportEphemeralOS\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"The Virtual Machine size Standard_D8s_v6 does not support Ephemeral OS disk.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  },\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j5164032-svc/providers/Microsoft.ContainerService/managedClusters/prow-j5164032-svc/agentPools/usd8sv63\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"VMSizeDoesNotSupportEphemeralOS\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"The Virtual Machine size Standard_D8s_v6 does not support Ephemeral OS disk.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7e0517f6f5854897e68fc282cbc719cf9446dd7a6eff1bd4ae096575dc4519bf","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VMSizeDoesNotSupportEphemeralOS; detail message The Virtual Machine size Standard_D8s_v6 does not support Ephemeral OS disk.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7e0517f6f5854897e68fc282cbc719cf9446dd7a6eff1bd4ae096575dc4519bf"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037432888353361920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4641,"pr_state":"closed","pr_sha":"119265856cafc9dafafbabdf9105a8d7adfe4e00","final_merged_sha":"6a659af9d2b69b8b76999874fc48a3353e46370b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T07:33:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037431701516324864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4630,"pr_state":"closed","pr_sha":"74afc8ada7ec0c736c6d87ab161be0edb9b07e2d","final_merged_sha":"7e07b7c9223871a196c30b61870b1a0f20ef1025","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T07:29:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037428957577482240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4578,"pr_state":"closed","pr_sha":"a86bfef70cac6243de22358ad82090115c0d1835","final_merged_sha":"35d72d570de220208509fd109c0b0be041d2858a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T07:18:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037415842160840704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"46f4bbad637f9a7610067fae1e313a3134a8b287","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T06:26:04Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bae031f3dcfd311ff3f57516abca1a7375afa76193b8c90f03fc737d78df25df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037415842160840704","occurred_at":"2026-03-27T06:26:04Z","failed_at":"provision","signature_id":"f3d91865b3264b24dab97ef0c3d6f1a144a017e31850832d7553d96dec98be0a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-27T07:03:07.850Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-27T07:03:10.439Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build05.ci.openshift.org\\nSaved credentials for registry.build05.ci.openshift.org into /tmp/tmp.kWAyib6f7J/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build05.ci.openshift.org/ci-op-46rclhn2/pipeline@sha256:88e924de288539a9fed728b0359e0f818914537bfee27d713c28505e17d27593 to arohcpsvcdev.azurecr.io/ci-op-46rclhn2/pipeline:88e924de288539a9fed728b0359e0f818914537bfee27d713c28505e17d27593.\\nThe image will still be available under it's original digest sha256:88e924de288539a9fed728b0359e0f818914537bfee27d713c28505e17d27593 in the target registry.\\nError: Get \\\"https://registry.build05.ci.openshift.org/v2/ci-op-46rclhn2/pipeline/blobs/sha256:93e3f7447420757b055fed9b7beb4211e24fa0393cac6adcf81c50b1ad4a44fe\\\": read tcp 172.24.211.114:48578-\u003e18.116.110.190:443: read: connection reset by peer\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"25d0dbaadf0a411598f3d71f1b57491ac1deb806822d7cc366ca66153b1c2509","failure_pattern":"Error: Get \"\u003curl\u003e\": read tcp \u003cip\u003e-\u003e\u003cip\u003e: read: connection reset by peer","search_query":"Error: Get \\\"https://registry.build05.ci.openshift.org/v2/ci-op-46rclhn2/pipeline/blobs/sha256:93e3f7447420757b055fed9b7beb4211e24fa0393cac6adcf81c50b1ad4a44fe\\\": read tcp 172.24.211.114:48578-\u003e18.116.110.190:443: read: connection reset by peer"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["25d0dbaadf0a411598f3d71f1b57491ac1deb806822d7cc366ca66153b1c2509"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037413226223767552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4578,"pr_state":"closed","pr_sha":"a86bfef70cac6243de22358ad82090115c0d1835","final_merged_sha":"35d72d570de220208509fd109c0b0be041d2858a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-27T06:15:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6a0bf2dedad4f1a0f496cde66ca4fb34eac5a834c4d17cc7cd1eaa98b6f1737b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037413226223767552","occurred_at":"2026-03-27T06:15:41Z","failed_at":"provision","signature_id":"2c8b7bab177758fc6366ffc7bb70006e0c4eb50001f3ad2770879f7fe17a23f6","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-27T06:56:41.083Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-27T06:56:41.591Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:89e1e7a5-df6a-4580-ad52-95f7059215f3 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j3767552] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j3767552.hcpsvc.osadev.cloud] image:map[digest:sha256:646f4826e848ab0833b51037e38c5a0869e4d2bcd4defb8725ae94ae82a0ee66 registry:arohcpsvcdev.azurecr.io repository:ci-op-6xklzdls/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6xklzdls/pipeline:pull workloadIdentityClientId:77622d9f-9d39-430c-bcbb-1b69bf035b1e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:92133d4f-b986-4e3c-8d23-ebd6b649e36c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-27T06:56:41.591Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-27T06:56:42.037Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-27T06:56:42.324Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-27T06:56:42.324Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-27T06:56:42.436Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-27T06:56:48.769Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.10:15017, code 500: 500 Internal Server Error\"\ntime=2026-03-27T06:56:48.769Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-27T06:56:48.769Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-27T06:56:50.562Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.10:15017, code 500: 500 Internal Server Error\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:89e1e7a5-df6a-4580-ad52-95f7059215f3 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j3767552] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j3767552.hcpsvc.osadev.cloud] image:map[digest:sha256:646f4826e848ab0833b51037e38c5a0869e4d2bcd4defb8725ae94ae82a0ee66 registry:arohcpsvcdev.azurecr.io repository:ci-op-6xklzdls/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6xklzdls/pipeline:pull workloadIdentityClientId:77622d9f-9d39-430c-bcbb-1b69bf035b1e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:92133d4f-b986-4e3c-8d23-ebd6b649e36c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-27T06:56:50.567Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6SUTU8bPRDH7/sprJyIFEcoz1NaUXGgUKlSpSoCTlQ9ON4/uwavZzszBoH48JUJL9sQCG1vzuQ3/3ndqSLUMIQye4jZM7VTp24RsUWLc3j9GlK9K8ohNROzNH1zHR5NuES6s0jv/IN5/L0yxpjRMfgyeIwmZiQQCZQap89/LuH9rC1xuHEaKM0pBn9dSBcjXdkOysGLlX/USaRtSM0Lrl9OTuZHlNdIr8UPsih1R/fdO8RZSKEEHeAyHfhNHZNtfT91N5kx9dQV8lEtZlHwEcXVePb+bTuXXIMOSYeOxeNTSPXzuqynpEwxgu1iLfFnGhGuBltE+FLnBs1D9JGuH7Ld3M4XeevBGs6CL2/GGUNa8N1QL8QG0UA2pIYh8rgEnuc5xkFJfY5xUxPwM0O0LBCSlnC/D7NZxr+DrMvavqBzDM/QOdNlqMEH0ZW8Vibq+1fzv/9y9r2n/Nb+rducp+H9ldPqxFdFqh8fq4u8ACco5HO5BVLdmqsWDKOhg6jrerOAXgHJbNVOUcxbo9n2bMdu/2dn70+2d3bf7ez+Pzsdjc10al5jPpyOxuPq1pxTSOYipHovpAQeXDBK5ulwDS/W6qmqbk3PVP59SnRieqqX8NtEJiZSMzEdRFxTJIlrsFlcD4qvIf5XAAAA//86ZdTgZwUAAA==\"\ntime=2026-03-27T06:56:50.570Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-27T06:56:50.570Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.10:15017, code 500: 500 Internal Server Error\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.10:15017, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037325355399778304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"4337c17161a9ffcc348c2be390c546093439d358","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-27T00:26:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}