{"meta":{"date":"2026-02-27","timezone":"UTC","generated_at":"2026-11-02T03:03:31Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":86,"failed_runs":43,"runs_with_occurrences":43,"runs_with_failure_pattern_matches":43,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":43,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027529723965345792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T23:42:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e0d9d817c69c4e9499f311d5f6dbaee225d839b43f751d054b9a7260fc2826f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027529723965345792","occurred_at":"2026-02-27T23:42:10Z","failed_at":"provision","signature_id":"0d28b12b0aefa50aeef54836f8e8ede62debaa59cb86599f1efd132c6b7fe930","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-28T00:49:28.986Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-28T00:49:29.497Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:617a3658-19d6-4b12-b380-1c80483e8cde cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j5345792] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j5345792.hcpsvc.osadev.cloud] image:map[digest:sha256:2146b3b62b67d23bd7d01d1994f1cd266a3f4b678028fcaa597a0e3c6a4a2c63 registry:arohcpsvcdev.azurecr.io repository:ci-op-xy962icx/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-xy962icx/pipeline:pull workloadIdentityClientId:5272fcbe-2c7c-4e97-a9a0-30612db4264e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:6da9dbef-3ae0-4d62-8caf-92d80374b73a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-02-28T00:49:29.497Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-28T00:49:29.684Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-02-28T00:49:29.984Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-02-28T00:49:29.984Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-28T00:49:30.113Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-28T00:49:36.677Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.5:15017, code 500: 500 Internal Server Error\"\ntime=2026-02-28T00:49:36.677Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-28T00:49:36.677Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-28T00:49:38.315Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.5:15017, code 500: 500 Internal Server Error\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:617a3658-19d6-4b12-b380-1c80483e8cde cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j5345792] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j5345792.hcpsvc.osadev.cloud] image:map[digest:sha256:2146b3b62b67d23bd7d01d1994f1cd266a3f4b678028fcaa597a0e3c6a4a2c63 registry:arohcpsvcdev.azurecr.io repository:ci-op-xy962icx/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-xy962icx/pipeline:pull workloadIdentityClientId:5272fcbe-2c7c-4e97-a9a0-30612db4264e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:6da9dbef-3ae0-4d62-8caf-92d80374b73a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-02-28T00:49:38.318Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xUTU/cMBC951dYObHSOlptqwqoOPBRqVKlagWcqHpwnEdicDzpeAwC8eOrsHxEIezuzR6/eTPvjTWZhyhGpMQWUR2pyogRU3rsUXkDK79cqA6jsAv1XK1Dv02LtxDuEJ4jsTP2NTz7kymlVH6GztNDiyD5XOURMToKtRF8uG7BawsWd+1sf2ZcM2ID7kHmNmoXxZF2oWbE+Mp1nKQhdo9GHIUVeWcfnvHe0/04Y7KZc/xLiNLzIEhf2lEY98VrkDZJmk94LmAZsmK6cxX41JsPFXW03UYtF+A7Z3FsLaVdvTxNUag9fxntGa5dcCMFsRjkFYZJN7YrzGNiFJbaHvlmBvlxIW0pCJP360FMObgxSXuYCqzhYSesfZuj5VXy/sSFyoW6B3XJe12+36eSNg0/kDSfp/68vFydU5rwddpmn6KAp6S+nHVrgqnx+quH5pxMihh6tFnmbhwjn7dwvvy1Leqzv9+z21SCAwTxR78EYvak7hswlLgWUUzbqRJyDwS1VxlBH97Ll4vlN71Y6uX+5WJx+PXgcHlwlc9UUagNmC/7V/lslj2pG3JB3bpQHbkQwIPVRUG9b6zhqhrvqOxJdUz963ujc9VRtQbvRjJXnuq5ahGjqXtK4gqsyoeB+ArR/g8AAP//z4pdV2AFAAA=\"\ntime=2026-02-28T00:49:38.320Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-28T00:49:38.320Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.5:15017, code 500: 500 Internal Server Error\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.5:15017, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027505282791247872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4247,"pr_state":"closed","pr_sha":"18cc623e43897e1df5fc8b18af2409a3bc71d000","final_merged_sha":"18cc623e43897e1df5fc8b18af2409a3bc71d000","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T22:05:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4259/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027500006361010176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4259,"pr_state":"closed","pr_sha":"6f427a2276ceaf4cb0ac19b460588b2bfefed930","final_merged_sha":"6f427a2276ceaf4cb0ac19b460588b2bfefed930","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T21:44:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027494826986967040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4258,"pr_state":"closed","pr_sha":"5906031723a0abcd54be256846732b88f0afffdb","final_merged_sha":"e2023dafc2500edee1f446eac424284b5b9df4e3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T21:23:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1b2ad8c31896977ed93e4bb3b19b69bd30a3d34b78fcddf3fbe7525754ee108f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027494826986967040","occurred_at":"2026-02-27T21:23:30Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T21:55:15.612Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T21:55:17.994Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T21:55:19.119Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2f9bc3df8438c699ca6c98988ce60c2ccb66ddaa1aba21bdba0c58ab4c7a01d2\ntime=2026-02-27T21:57:20.128Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6967040/providers/Microsoft.Resources/deployments/2f9bc3df8438c699ca6c98988ce60c2ccb66ddaa1aba21bdba0c58ab4c7a01d2/operationStatuses/08584293775670295238\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6967040/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6967040/providers/Microsoft.Monitor/accounts/services-j6967040/providers/Microsoft.Authorization/roleAssignments/ee0c365c-7e98-5500-84a5-ba6bd06cc809\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260227T215540Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'ea5e72a9-bedd-48b5-863d-1708d3c4ce83', request correlation id '5761150a-e097-4e6f-be30-8d18b690b860'.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["301528d535b97ee60ce2db8b97bca821eb32cd4fca51b7a3da578d2b4d7fdf33"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027487114957950976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4074,"pr_state":"closed","pr_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","final_merged_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T20:52:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027486010853560320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"open","pr_sha":"e2d5af5e492cd046c73142c048211ddfa514083f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T20:48:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027485053633695744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3880,"pr_state":"open","pr_sha":"7b88114939e6d13d5868f702667bcfcac8d7c2da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T20:44:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4256/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027482968318021632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4256,"pr_state":"closed","pr_sha":"bbafa171ba8d0a43a6eb8c9dd370352fe356cd53","final_merged_sha":"bbafa171ba8d0a43a6eb8c9dd370352fe356cd53","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T20:36:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027478792666091520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4255,"pr_state":"open","pr_sha":"7ec0c2cf1ecf3837013224577c87c1158a02c4ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T20:19:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"673fd460dd2075c4b6965a70431ba48010549f8f84332f7f108fa2ca9c6455a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027478792666091520","occurred_at":"2026-02-27T20:19:47Z","failed_at":"e2e","signature_id":"369eb794f0979396406b27d0c1bd2548822ea8d3be9906da6d77c8fc25ae0e90","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00080c8a0\u003e: \n    failed to create HCP cluster list-test-cluster-f2mhhl: failed waiting for cluster=\"list-test-cluster-f2mhhl\" in resourcegroup=\"cluster-listing-vjlprk\" to finish creating: GET https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\",\n      \"name\": \"11c25517-87e3-428b-9690-af763d4e7708\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T21:16:02.290567422Z\",\n      \"endTime\": \"2026-02-27T21:16:59.254117768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster list-test-cluster-f2mhhl: failed waiting for cluster=\\\"list-test-cluster-f2mhhl\\\" in resourcegroup=\\\"cluster-listing-vjlprk\\\" to finish creating: GET https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"name\\\": \\\"11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T21:16:02.290567422Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T21:16:59.254117768Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00080c880\u003e{\n            msg: \"failed waiting for cluster=\\\"list-test-cluster-f2mhhl\\\" in resourcegroup=\\\"cluster-listing-vjlprk\\\" to finish creating: GET https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"name\\\": \\\"11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T21:16:02.290567422Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T21:16:59.254117768Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00080c8a0\u003e: \n    failed to create HCP cluster list-test-cluster-f2mhhl: failed waiting for cluster=\"list-test-cluster-f2mhhl\" in resourcegroup=\"cluster-listing-vjlprk\" to finish creating: GET https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\",\n      \"name\": \"11c25517-87e3-428b-9690-af763d4e7708\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T21:16:02.290567422Z\",\n      \"endTime\": \"2026-02-27T21:16:59.254117768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster list-test-cluster-f2mhhl: failed waiting for cluster=\\\"list-test-cluster-f2mhhl\\\" in resourcegroup=\\\"cluster-listing-vjlprk\\\" to finish creating: GET https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"name\\\": \\\"11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T21:16:02.290567422Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T21:16:59.254117768Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00080c880\u003e{\n            msg: \"failed waiting for cluster=\\\"list-test-cluster-f2mhhl\\\" in resourcegroup=\\\"cluster-listing-vjlprk\\\" to finish creating: GET https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"name\\\": \\\"11c25517-87e3-428b-9690-af763d4e7708\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T21:16:02.290567422Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T21:16:59.254117768Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-235/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027478557411774464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"c1d21b6487076a17d3a8e6f000bc9e39ec13658f","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T20:18:51Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"38656610f7c99079e59001e16dd10f08d1191ce886b5d142f4d36f1305138683","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027478557411774464","occurred_at":"2026-02-27T20:18:51Z","failed_at":"e2e","signature_id":"4ccc5d91d3499810cbaff71f6ee5bf664cf53765ab52bb1c987aff66c635c434","test_name":"Customer should not be able to create a 4.18 HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_missing_info.go:82]: Expected\n    \u003c*fmt.wrapError | 0xc000e821a0\u003e: \n    failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \"illegal-hcp-cluster\" in resourcegroup=\"illegal-ocp-version-22v79l\": PUT https://rp.j1774464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/illegal-ocp-version-22v79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/illegal-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.18\\\": must be at least 4.19\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-22v79l\\\": PUT https://rp.j1774464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/illegal-ocp-version-22v79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/illegal-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.18\\\\\\\": must be at least 4.19\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e82180\u003e{\n            msg: \"failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-22v79l\\\": PUT https://rp.j1774464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/illegal-ocp-version-22v79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/illegal-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.18\\\\\\\": must be at least 4.19\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000cfe360\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"390ce0b7-71ee-4408-8fbb-5e59f0083c6f\",\n                        ],\n                        \"Date\": [\n                            \"Fri, 27 Feb 2026 21:34:35 GMT\",\n                        ],\n                        \"Content-Length\": [\"176\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"29\"],\n                        \"Server\": [\"istio-envoy\"],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000e820a0\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 32, 118, 97, 108, 117, 101, 58, 32, 92, 34, 52, 46, 49, 56, 92, 34, 58, 32, 109, 117, 115, 116, 32, 98, 101, 32, 97, 116, 32, 108, 101, 97, 115, 116, 32, 52, 46, 49, 57, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 118, 101, 114, 115, 105, 111, 110, 46, 105, 100, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                        i: 0,\n                    },\n                    ContentLength: 176,\n                    TransferEncoding: nil,\n                    Close: false,\n                    Uncompressed: false,\n                    Trailer: nil,\n                    Request: {\n                        Me...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto match error\n    \u003c*matchers.MatchRegexpMatcher | 0xc000cfe3f0\u003e: {\n        Regexp: \"Version .* (doesn't exist|is disabled)\",\n        Args: nil,\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_missing_info.go:82]: Expected\n    \u003c*fmt.wrapError | 0xc000e821a0\u003e: \n    failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \"illegal-hcp-cluster\" in resourcegroup=\"illegal-ocp-version-22v79l\": PUT https://rp.j1774464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/illegal-ocp-version-22v79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/illegal-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.18\\\": must be at least 4.19\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster illegal-hcp-cluster: failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-22v79l\\\": PUT https://rp.j1774464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/illegal-ocp-version-22v79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/illegal-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.18\\\\\\\": must be at least 4.19\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e82180\u003e{\n            msg: \"failed starting cluster creation \\\"illegal-hcp-cluster\\\" in resourcegroup=\\\"illegal-ocp-version-22v79l\\\": PUT https://rp.j1774464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/illegal-ocp-version-22v79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/illegal-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.18\\\\\\\": must be at least 4.19\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000cfe360\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"390ce0b7-71ee-4408-8fbb-5e59f0083c6f\",\n                        ],\n                        \"Date\": [\n                            \"Fri, 27 Feb 2026 21:34:35 GMT\",\n                        ],\n                        \"Content-Length\": [\"176\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"29\"],\n                        \"Server\": [\"istio-envoy\"],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000e820a0\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 32, 118, 97, 108, 117, 101, 58, 32, 92, 34, 52, 46, 49, 56, 92, 34, 58, 32, 109, 117, 115, 116, 32, 98, 101, 32, 97, 116, 32, 108, 101, 97, 115, 116, 32, 52, 46, 49, 57, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 118, 101, 114, 115, 105, 111, 110, 46, 105, 100, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                        i: 0,\n                    },\n                    ContentLength: 176,\n                    TransferEncoding: nil,\n                    Close: false,\n                    Uncompressed: false,\n                    Trailer: nil,\n                    Request: {\n                        Me...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto match error\n    \u003c*matchers.MatchRegexpMatcher | 0xc000cfe3f0\u003e: {\n        Regexp: \"Version .* (doesn't exist|is disabled)\",\n        Args: nil,\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6c909a60aa3806b40941c3a45de8a216704a5dee354d1a9140ea6ea9e7f0d2d","failure_pattern":"Version .* (doesn't exist|is disabled)","search_query":"Version .* (doesn't exist|is disabled)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6c909a60aa3806b40941c3a45de8a216704a5dee354d1a9140ea6ea9e7f0d2d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472930593574912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4249,"pr_state":"closed","pr_sha":"67bc090ef1f250bde852b94552a8cfc7fe13e836","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T19:56:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ca389fa2cbed4ae13c519f2a55355f348a79d1a10a7c531b2cdf2a48be905e2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472930593574912","occurred_at":"2026-02-27T19:56:29Z","failed_at":"provision","signature_id":"538cde4a94c880730ee7d7ca3868c5dc74f5fc70032636c11f21abbcf6ea464b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T20:53:18.564Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T20:53:38.885Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.4TIpASyiGq/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-9q4dtjgy/pipeline@sha256:57661c7cefbf66d8260d9118c9d7ec05c3dd9a3b3f010b89058783c7c8d85f36 to arohcpsvcdev.azurecr.io/ci-op-9q4dtjgy/pipeline:57661c7cefbf66d8260d9118c9d7ec05c3dd9a3b3f010b89058783c7c8d85f36.\\nThe image will still be available under it's original digest sha256:57661c7cefbf66d8260d9118c9d7ec05c3dd9a3b3f010b89058783c7c8d85f36 in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472906090450944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4250,"pr_state":"closed","pr_sha":"2a8e06eb3c9050ea4d800ebbac30cfc698f79655","final_merged_sha":"2a8e06eb3c9050ea4d800ebbac30cfc698f79655","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T19:56:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4251/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472869637754880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4251,"pr_state":"closed","pr_sha":"b431563dd1cc0fb4edd2381eef7b71245371f32c","final_merged_sha":"b431563dd1cc0fb4edd2381eef7b71245371f32c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T19:56:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4252/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472846757826560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4252,"pr_state":"closed","pr_sha":"69935bdeb534d94d14836a80c93e29f338f4c9bf","final_merged_sha":"69935bdeb534d94d14836a80c93e29f338f4c9bf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T19:56:09Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ac9f7597324fb6bb59d7a5e9c7b14006d1907e6b7dd6c8ea994f7e72c7746db4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4252/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472846757826560","occurred_at":"2026-02-27T19:56:09Z","failed_at":"e2e","signature_id":"329cd7f9d58e125265f519d19325faa6bf4b421c6a2aba93adff32287e8832dd","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:131]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000984840\u003e: \n    failed to create route: the server is currently unable to handle the request\n    {\n        msg: \"failed to create route: the server is currently unable to handle the request\",\n        err: \u003c*errors.StatusError | 0xc00103cc80\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"the server is currently unable to handle the request\",\n                Reason: \"ServiceUnavailable\",\n                Details: {\n                    Name: \"\",\n                    Group: \"\",\n                    Kind: \"\",\n                    UID: \"\",\n                    Causes: [\n                        {\n                            Type: \"UnexpectedServerResponse\",\n                            Message: \"error trying to reach service: tls: first record does not look like a TLS handshake\",\n                            Field: \"\",\n                        },\n                    ],\n                    RetryAfterSeconds: 0,\n                },\n                Code: 503,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:131]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000984840\u003e: \n    failed to create route: the server is currently unable to handle the request\n    {\n        msg: \"failed to create route: the server is currently unable to handle the request\",\n        err: \u003c*errors.StatusError | 0xc00103cc80\u003e{\n            ErrStatus: {\n                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                ListMeta: {\n                    SelfLink: \"\",\n                    ResourceVersion: \"\",\n                    Continue: \"\",\n                    RemainingItemCount: nil,\n                },\n                Status: \"Failure\",\n                Message: \"the server is currently unable to handle the request\",\n                Reason: \"ServiceUnavailable\",\n                Details: {\n                    Name: \"\",\n                    Group: \"\",\n                    Kind: \"\",\n                    UID: \"\",\n                    Causes: [\n                        {\n                            Type: \"UnexpectedServerResponse\",\n                            Message: \"error trying to reach service: tls: first record does not look like a TLS handshake\",\n                            Field: \"\",\n                        },\n                    ],\n                    RetryAfterSeconds: 0,\n                },\n                Code: 503,\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60","failure_pattern":"failed to create route: the server is currently unable to handle the request","search_query":"failed to create route: the server is currently unable to handle the request"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["eaacc54ebd46cefe15abd433d5c3a394c2c9eab33016bf077e0076e05e8a3f60"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4253/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027472823060008960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4253,"pr_state":"closed","pr_sha":"633f9aff90dcd1f163d22e121a4d57be9f98ceaa","final_merged_sha":"633f9aff90dcd1f163d22e121a4d57be9f98ceaa","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T19:56:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027454675485200384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4074,"pr_state":"closed","pr_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","final_merged_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T18:43:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453961157480448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4122,"pr_state":"closed","pr_sha":"01e7af77a9faabb5b15aad7f797bad3d64fc2f70","final_merged_sha":"01e7af77a9faabb5b15aad7f797bad3d64fc2f70","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T18:41:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453458734387200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4248,"pr_state":"closed","pr_sha":"36daae304273e490748d831d4139277d5e16eaab","final_merged_sha":"36daae304273e490748d831d4139277d5e16eaab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T18:39:07Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1e4b9dac702de5ef39df9aa7d1061d22321c479aac829ab478e84d0c0f584a02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453458734387200","occurred_at":"2026-02-27T18:39:07Z","failed_at":"e2e","signature_id":"a813b03282db5943ff117ad7785e3272a640de827555530cedabaaca59e30a1e","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016f2090\u003e: \n    failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\n    failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000780e00\u003e{\n                msg: \"failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                err: \u003c*fmt.wrapError | 0xc000780de0\u003e{\n                    msg: \"failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                    err: \u003c*azidentity.AuthenticationFailedError | 0xc000e35800\u003e{\n                        RawResponse: nil,\n                        credType: \"AzureCLICredential\",\n                        message: \"signal: killed\",\n                        omitResponse: false,\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xc0008c4160\u003e{\n                msg: \"failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                err: \u003c*fmt.wrapError | 0xc0008c4140\u003e{\n                    msg: \"failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                    err: \u003c*azidentity.AuthenticationFailedError | 0xc00141a360\u003e{\n                        RawResponse: nil,\n                        credType: \"AzureCLICredential\",\n                        message: \"signal: killed\",\n                        omitResponse: false,\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016f2090\u003e: \n    failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\n    failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000780e00\u003e{\n                msg: \"failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                err: \u003c*fmt.wrapError | 0xc000780de0\u003e{\n                    msg: \"failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                    err: \u003c*azidentity.AuthenticationFailedError | 0xc000e35800\u003e{\n                        RawResponse: nil,\n                        credType: \"AzureCLICredential\",\n                        message: \"signal: killed\",\n                        omitResponse: false,\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xc0008c4160\u003e{\n                msg: \"failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                err: \u003c*fmt.wrapError | 0xc0008c4140\u003e{\n                    msg: \"failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed\",\n                    err: \u003c*azidentity.AuthenticationFailedError | 0xc00141a360\u003e{\n                        RawResponse: nil,\n                        credType: \"AzureCLICredential\",\n                        message: \"signal: killed\",\n                        omitResponse: false,\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c63bbcb9f99d2c64188ab8aa6dba590eb943720e17523ff6b409f91995877c19","failure_pattern":"failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed","search_query":"failed to search for managed resource groups: failed to list resource groups while discovering managed groups: AzureCLICredential: signal: killed"}},{"row_id":"73f94fb150cc1d2a18d7ff18512ca346b6cf4c0487015f1f16b2cd06170db16a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453458734387200","occurred_at":"2026-02-27T18:39:07Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"93ecfabbb5b409da46245885db2d7f56906577e3df02e2f76e090fe1f68f5530","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453458734387200","occurred_at":"2026-02-27T18:39:07Z","failed_at":"e2e","signature_id":"eeaa03d606dcd47d946a3350ecf9a6c74c18685e96288d153cebda966b68cd66","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:314]: Timed out after 300.004s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001304960\u003e: \n    subscription state not found\n    {\n        s: \"subscription state not found\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:314]: Timed out after 300.004s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001304960\u003e: \n    subscription state not found\n    {\n        s: \"subscription state not found\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6be2a1429c526e145930492292d4b47bfb07edefddf28cea615d751f6a2c96b1","failure_pattern":"subscription state not found","search_query":"subscription state not found"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6be2a1429c526e145930492292d4b47bfb07edefddf28cea615d751f6a2c96b1","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","c63bbcb9f99d2c64188ab8aa6dba590eb943720e17523ff6b409f91995877c19"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453273937547264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4122,"pr_state":"closed","pr_sha":"01e7af77a9faabb5b15aad7f797bad3d64fc2f70","final_merged_sha":"01e7af77a9faabb5b15aad7f797bad3d64fc2f70","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T18:38:23Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e0e47356cfdba688f368fc066d0dd64f6f55dc2a8a1d3bfeab9f9bcfcaa5df71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027453273937547264","occurred_at":"2026-02-27T18:38:23Z","failed_at":"e2e","signature_id":"79859d34ace84752d14d893a7bd19a88d0fc8059b65eca666d8b3ceb4b0fc3b6","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:314]: Timed out after 300.002s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001324950\u003e: \n    subscription state not found\n    {\n        s: \"subscription state not found\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:314]: Timed out after 300.002s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001324950\u003e: \n    subscription state not found\n    {\n        s: \"subscription state not found\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6be2a1429c526e145930492292d4b47bfb07edefddf28cea615d751f6a2c96b1","failure_pattern":"subscription state not found","search_query":"subscription state not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6be2a1429c526e145930492292d4b47bfb07edefddf28cea615d751f6a2c96b1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3913/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027449108360335360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3913,"pr_state":"open","pr_sha":"2841bf78b2a3642eec1fe39e1db0fc79dbb181fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T18:21:50Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"66ba62e53b6255a64c128f4e3d980755df2863de71736f445e2e11d4c0a42bb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3913/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027449108360335360","occurred_at":"2026-02-27T18:21:50Z","failed_at":"e2e","signature_id":"8a6e90c3cbe4d7e7ac5e42bcbff388ecaae10c64895d530be363c61a8d5de8e9","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.000s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0005261b0\u003e: \n    nfd-worker DaemonSet not found\n    {\n        s: \"nfd-worker DaemonSet not found\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.000s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0005261b0\u003e: \n    nfd-worker DaemonSet not found\n    {\n        s: \"nfd-worker DaemonSet not found\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb","failure_pattern":"nfd-worker DaemonSet not found","search_query":"nfd-worker DaemonSet not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4256/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027446090004959232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4256,"pr_state":"closed","pr_sha":"bbafa171ba8d0a43a6eb8c9dd370352fe356cd53","final_merged_sha":"bbafa171ba8d0a43a6eb8c9dd370352fe356cd53","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T18:09:50Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6116ab1c245566a6682ed6fdd8790a62b7096dad77cdc15163fa87d480a5eaf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4256/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027446090004959232","occurred_at":"2026-02-27T18:09:50Z","failed_at":"e2e","signature_id":"84799b6ebf73759f40cb780f8abc4f17e41ce904352248f840d0fd2793f58799","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00098b080\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nvdjt7\" to finish creating: GET https://rp.j4959232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\",\n      \"name\": \"f174e9bc-6287-4dd3-8749-d64aac8eaab5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T19:50:29.098569889Z\",\n      \"endTime\": \"2026-02-27T19:51:41.651819501Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-nvdjt7\\\" to finish creating: GET https://rp.j4959232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"name\\\": \\\"f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T19:50:29.098569889Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T19:51:41.651819501Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00098b040\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-nvdjt7\\\" to finish creating: GET https://rp.j4959232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"name\\\": \\\"f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T19:50:29.098569889Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T19:51:41.651819501Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00098b080\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nvdjt7\" to finish creating: GET https://rp.j4959232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\",\n      \"name\": \"f174e9bc-6287-4dd3-8749-d64aac8eaab5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T19:50:29.098569889Z\",\n      \"endTime\": \"2026-02-27T19:51:41.651819501Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-nvdjt7\\\" to finish creating: GET https://rp.j4959232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"name\\\": \\\"f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T19:50:29.098569889Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T19:51:41.651819501Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00098b040\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-nvdjt7\\\" to finish creating: GET https://rp.j4959232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"name\\\": \\\"f174e9bc-6287-4dd3-8749-d64aac8eaab5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T19:50:29.098569889Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T19:51:41.651819501Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027440180297732096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4244,"pr_state":"closed","pr_sha":"ced1ca31222cfd4db4ad2a0990260982e8f49da6","final_merged_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T17:46:21Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"823bf2a52a9f6e8f6d4c7f5943868eca71a9edd8c7bf28048bbb484c8e0cc75a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027440180297732096","occurred_at":"2026-02-27T17:46:21Z","failed_at":"provision","signature_id":"82a43932bbf62ef6d35b6d6d772bd6e0fbe1d7c8d2885d31dabeb7e0e9ecbb0c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-27T19:01:42.145Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T19:01:42.682Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj7732096.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj7732096.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:51cf59a2-0c5f-43c9-9547-21f7f88ae774 databaseName:arohcpprow-rp-j7732096 databaseUrl:https://arohcpprow-rp-j7732096.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-2gd5cq9w/pipeline@sha256:340f1ff0b72f84ade72c96e414a2eecdbd7189d3c39d2998e0ba31d934b09c0f replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-2gd5cq9w/pipeline:pull workloadIdentityClientId:dc589e05-8344-45ec-9b8f-177eb62874b1 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:51cf59a2-0c5f-43c9-9547-21f7f88ae774 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T19:01:42.682Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T19:01:43.074Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T19:01:43.501Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T19:01:43.501Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T19:01:43.625Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T19:01:43.931Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"backend/templates/backend.deployment.yaml:68:25\\n  executing \\\"backend/templates/backend.deployment.yaml\\\" at \u003c.Values.miMock.certName\u003e:\\n    nil pointer evaluating interface {}.certName\"\ntime=2026-02-27T19:01:43.931Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T19:01:43.931Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T19:01:44.052Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"failed to get status for release aro-hcp-backend-dev: release: not found\"\ntime=2026-02-27T19:01:44.052Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T19:01:44.053Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: backend/templates/backend.deployment.yaml:68:25\\n  executing \\\"backend/templates/backend.deployment.yaml\\\" at \u003c.Values.miMock.certName\u003e:\\n    nil pointer evaluating interface {}.certName\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68d91604f3ad122a1d62133a42862f285cc314b8eee8ff4c3bfc85746defdd22","failure_pattern":"backend/templates/backend.deployment.yaml:68:25 executing \"backend/templates/backend.deployment.yaml\" at \u003c.Values.miMock.certName\u003e: nil pointer evaluating interface {}.certName","search_query":"time=2026-02-27T12:19:50.449Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"failed to get status for release aro-h"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["68d91604f3ad122a1d62133a42862f285cc314b8eee8ff4c3bfc85746defdd22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027439492310241280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"f743ff17fb2145a02ae8d79d65f69382706ab53d","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T17:43:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027435726706053121","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"35f2d8f4e5341e8214b950098a9a81bcf4557ea3","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T17:28:39Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4c212bb2ba4b91dae90bd0abb01994c0c876e544ab162b5c15b370446d9fb419","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027435726706053121","occurred_at":"2026-02-27T17:28:39Z","failed_at":"provision","signature_id":"c4abb5727567e8fcea5d580c742ad573474817c024ecbc80cb9028e193c384ab","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T18:28:42.331Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T18:29:17.612Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.SZzAf9qcGP/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-ndhix1pm/pipeline@sha256:3df5e467028465e193fd400e34fca415dc7d9609e11dbc7c33ca69edb14a7e7d to arohcpsvcdev.azurecr.io/ci-op-ndhix1pm/pipeline:3df5e467028465e193fd400e34fca415dc7d9609e11dbc7c33ca69edb14a7e7d.\\nThe image will still be available under it's original digest sha256:3df5e467028465e193fd400e34fca415dc7d9609e11dbc7c33ca69edb14a7e7d in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027433003847782400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4254,"pr_state":"closed","pr_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","final_merged_sha":"1ffff54f1d2e647ca0e3bb96102f12280f7605cf","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T17:17:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027430912441978880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T17:09:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4253/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027426571421224960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4253,"pr_state":"closed","pr_sha":"633f9aff90dcd1f163d22e121a4d57be9f98ceaa","final_merged_sha":"633f9aff90dcd1f163d22e121a4d57be9f98ceaa","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T16:52:16Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d27a0b69073d28158531e8c608ac1db684884714c866ea88e17d05c8fe04b4d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4253/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027426571421224960","occurred_at":"2026-02-27T16:52:16Z","failed_at":"e2e","signature_id":"3089ff315cb6ad41ea435986a3a48987882d5de0e7e963a3e20feb0c623e219b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.002s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00081e090\u003e: \n    nfd-worker DaemonSet not found\n    {\n        s: \"nfd-worker DaemonSet not found\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.002s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00081e090\u003e: \n    nfd-worker DaemonSet not found\n    {\n        s: \"nfd-worker DaemonSet not found\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb","failure_pattern":"nfd-worker DaemonSet not found","search_query":"nfd-worker DaemonSet not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4252/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027425468193443840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4252,"pr_state":"closed","pr_sha":"69935bdeb534d94d14836a80c93e29f338f4c9bf","final_merged_sha":"69935bdeb534d94d14836a80c93e29f338f4c9bf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T16:47:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2a418f6015054591d6c63f92b301a2afb2f47824b1e8fac9b1eace2e1cc9b961","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4252/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027425468193443840","occurred_at":"2026-02-27T16:47:53Z","failed_at":"e2e","signature_id":"78cd6760e945c1eef5d23b58535aedd22187a2d0388aa3a9c7a262679ac0d3c8","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.002s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000bd2180\u003e: \n    nfd-worker DaemonSet not found\n    {\n        s: \"nfd-worker DaemonSet not found\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.002s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000bd2180\u003e: \n    nfd-worker DaemonSet not found\n    {\n        s: \"nfd-worker DaemonSet not found\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb","failure_pattern":"nfd-worker DaemonSet not found","search_query":"nfd-worker DaemonSet not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c23bffe4fc38fc81c8d082825e89fc13c7c999cc0d8483fe137d227f4abf76fb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4251/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027424470200750080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4251,"pr_state":"closed","pr_sha":"b431563dd1cc0fb4edd2381eef7b71245371f32c","final_merged_sha":"b431563dd1cc0fb4edd2381eef7b71245371f32c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T16:43:56Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7652598b5541f1fd6cf00696b812aa8cb60b664fd447dec7ade243c8f08452bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4251/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027424470200750080","occurred_at":"2026-02-27T16:43:56Z","failed_at":"e2e","signature_id":"befca6f33c1e0b6feb867ce3c330733d247cbb9b5a575dd806591ec2301b765a","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.002s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001134410\u003e: \n    nfd-worker DaemonSet found but not ready: desired=2, ready=0\n    {\n        s: \"nfd-worker DaemonSet found but not ready: desired=2, ready=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.002s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001134410\u003e: \n    nfd-worker DaemonSet found but not ready: desired=2, ready=0\n    {\n        s: \"nfd-worker DaemonSet found but not ready: desired=2, ready=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db2106f50602e86b0820ab5d8f1364c6244d2c6b9eadd1761b98923fcbeb711a","failure_pattern":"nfd-worker DaemonSet found but not ready: desired=2, ready=0","search_query":"nfd-worker DaemonSet found but not ready: desired=2, ready=0"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["db2106f50602e86b0820ab5d8f1364c6244d2c6b9eadd1761b98923fcbeb711a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027423598267863040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4250,"pr_state":"closed","pr_sha":"2a8e06eb3c9050ea4d800ebbac30cfc698f79655","final_merged_sha":"2a8e06eb3c9050ea4d800ebbac30cfc698f79655","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T16:40:28Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"71125150e87740c39784bfa8ead01d9cefdbf4026cf481221c5accfd0793608c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027423598267863040","occurred_at":"2026-02-27T16:40:28Z","failed_at":"e2e","signature_id":"e815959fcb3d443b506a91289d4884b56cbb4195dbad011095280434428b9f00","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000931a60\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-x7fl2l\" to finish creating: GET https://rp.j7863040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\",\n      \"name\": \"6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T18:04:25.317040147Z\",\n      \"endTime\": \"2026-02-27T18:05:09.102514272Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-x7fl2l\\\" to finish creating: GET https://rp.j7863040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"name\\\": \\\"6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T18:04:25.317040147Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T18:05:09.102514272Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000931a40\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-x7fl2l\\\" to finish creating: GET https://rp.j7863040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"name\\\": \\\"6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T18:04:25.317040147Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T18:05:09.102514272Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000931a60\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-x7fl2l\" to finish creating: GET https://rp.j7863040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\",\n      \"name\": \"6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T18:04:25.317040147Z\",\n      \"endTime\": \"2026-02-27T18:05:09.102514272Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-x7fl2l\\\" to finish creating: GET https://rp.j7863040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"name\\\": \\\"6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T18:04:25.317040147Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T18:05:09.102514272Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000931a40\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-x7fl2l\\\" to finish creating: GET https://rp.j7863040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"name\\\": \\\"6cc44886-65c2-4ee1-b2c2-f7dbbcb26cb7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T18:04:25.317040147Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T18:05:09.102514272Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-227/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc1c67013e3bda1b19d39a88d7582aa2ec0ba7d2f460b8586b359f472717d21b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027423598267863040","occurred_at":"2026-02-27T16:40:28Z","failed_at":"e2e","signature_id":"baf09f18e23277fe5957c4ed512d8533f8fafb97d1ec366d1e8b0f360b94afd2","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.006s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000a394e0\u003e: \n    nfd-worker DaemonSet found but not ready: desired=2, ready=0\n    {\n        s: \"nfd-worker DaemonSet found but not ready: desired=2, ready=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:356]: Timed out after 300.006s.\nNFD worker DaemonSet should be created and have ready pods\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000a394e0\u003e: \n    nfd-worker DaemonSet found but not ready: desired=2, ready=0\n    {\n        s: \"nfd-worker DaemonSet found but not ready: desired=2, ready=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db2106f50602e86b0820ab5d8f1364c6244d2c6b9eadd1761b98923fcbeb711a","failure_pattern":"nfd-worker DaemonSet found but not ready: desired=2, ready=0","search_query":"nfd-worker DaemonSet found but not ready: desired=2, ready=0"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","db2106f50602e86b0820ab5d8f1364c6244d2c6b9eadd1761b98923fcbeb711a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027422758857609216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4249,"pr_state":"closed","pr_sha":"67bc090ef1f250bde852b94552a8cfc7fe13e836","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T16:37:08Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"aaa881b9985381c088276ce25794fee86e552047d90d01c256f3479108f8e6f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027422758857609216","occurred_at":"2026-02-27T16:37:08Z","failed_at":"provision","signature_id":"64edb330820d298cff2b90aa4de1f83808d751cc6c86e9b4faa154d5b35c37d2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T17:27:05.933Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T17:27:35.176Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.CaGoNhcOSj/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-d7502wz7/pipeline@sha256:1161d97dfab7c0d32936bfe9ac3421c623832b1a0b40803a3c69bc1d5f8668db to arohcpsvcdev.azurecr.io/ci-op-d7502wz7/pipeline:1161d97dfab7c0d32936bfe9ac3421c623832b1a0b40803a3c69bc1d5f8668db.\\nThe image will still be available under it's original digest sha256:1161d97dfab7c0d32936bfe9ac3421c623832b1a0b40803a3c69bc1d5f8668db in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027422437938827264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4246,"pr_state":"closed","pr_sha":"ab82d5ad402ec38c63584b6be7cef8d087ba8400","final_merged_sha":"ab82d5ad402ec38c63584b6be7cef8d087ba8400","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T16:35:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027421995171319808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4248,"pr_state":"closed","pr_sha":"36daae304273e490748d831d4139277d5e16eaab","final_merged_sha":"36daae304273e490748d831d4139277d5e16eaab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T16:34:05Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6abe6d9a25adf0ad5555f4bd5c6f2930a8250767d59ccc62d605169b1d461f5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027421995171319808","occurred_at":"2026-02-27T16:34:05Z","failed_at":"e2e","signature_id":"14e3471b7b190bc6f3dda8d3e1200b5893bba7d739c941f69cee8505e38a71e4","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:366]: Timed out after 300.002s.\nNFD worker images from registry.redhat.io should be pulled successfully with the added pull secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001515460\u003e: \n    image pull errors detected:\n    pod nfd-gc-685d666965-wbsn5 container nfd-gc: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\n    pod nfd-worker-j5chf container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\n    pod nfd-worker-wzvp9 container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\n    {\n        s: \"image pull errors detected:\\npod nfd-gc-685d666965-wbsn5 container nfd-gc: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\\npod nfd-worker-j5chf container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\\npod nfd-worker-wzvp9 container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:366]: Timed out after 300.002s.\nNFD worker images from registry.redhat.io should be pulled successfully with the added pull secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001515460\u003e: \n    image pull errors detected:\n    pod nfd-gc-685d666965-wbsn5 container nfd-gc: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\n    pod nfd-worker-j5chf container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\n    pod nfd-worker-wzvp9 container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\n    {\n        s: \"image pull errors detected:\\npod nfd-gc-685d666965-wbsn5 container nfd-gc: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\\npod nfd-worker-j5chf container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\\npod nfd-worker-wzvp9 container nfd-worker: ErrImagePull - unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a67252556c8280457e593412e689ee36106c42f07f222be680374eedb6db310","failure_pattern":"pod nfd-gc-685d666965-\u003cid\u003e container nfd-gc: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: copying system image from manifest","search_query":"pod nfd-gc-685d666965-wbsn5 container nfd-gc: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/openshift4/ose-node-feature-discovery:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: copying system image from manifest list: reading signatures: reading signature from https://registry.redhat.io/containers/sigstore/openshift4/ose-node-feature-discovery@sha256=f061573910f11a78a8c4e2cb636fcbd280a5a50962b2d51ab33ea42e3a2a3f96/signature-7: received unexpected HTTP status: 500 Internal Server Error; artifact err: provided artifact is a container image"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a67252556c8280457e593412e689ee36106c42f07f222be680374eedb6db310"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4235/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027420817360424960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4235,"pr_state":"open","pr_sha":"fea5b77c9fb0f5ab605e84f05f5890edc8162210","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T16:29:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027419463913050112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4237,"pr_state":"closed","pr_sha":"9cf9d0988b78e3d09f927214bb2299ea73c51d4c","final_merged_sha":"9cf9d0988b78e3d09f927214bb2299ea73c51d4c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T16:24:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4247/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027418861514526720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4247,"pr_state":"closed","pr_sha":"18cc623e43897e1df5fc8b18af2409a3bc71d000","final_merged_sha":"18cc623e43897e1df5fc8b18af2409a3bc71d000","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T16:21:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027411630349881344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4074,"pr_state":"closed","pr_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","final_merged_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T15:52:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4122/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027409726026813440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4122,"pr_state":"closed","pr_sha":"01e7af77a9faabb5b15aad7f797bad3d64fc2f70","final_merged_sha":"01e7af77a9faabb5b15aad7f797bad3d64fc2f70","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T15:45:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027403073491570688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4246,"pr_state":"closed","pr_sha":"ab82d5ad402ec38c63584b6be7cef8d087ba8400","final_merged_sha":"ab82d5ad402ec38c63584b6be7cef8d087ba8400","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T15:18:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7df2b0b1a4640aee29b9de37817e833cd5f33889a5c43ceacb3ac55667067de0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027403073491570688","occurred_at":"2026-02-27T15:18:54Z","failed_at":"provision","signature_id":"87be5682bc525d75e3d17c14d985a24d9614487c8ef58b871a5d132c852905ed","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T16:19:15.939Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T16:19:49.302Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.vU7Kvs5nkv/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-3kdmfil7/pipeline@sha256:0ddb47878b75a6f698580a63e387c553078a3d06576482db0ac6af213987e832 to arohcpsvcdev.azurecr.io/ci-op-3kdmfil7/pipeline:0ddb47878b75a6f698580a63e387c553078a3d06576482db0ac6af213987e832.\\nThe image will still be available under it's original digest sha256:0ddb47878b75a6f698580a63e387c553078a3d06576482db0ac6af213987e832 in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3913/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027402913919275008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3913,"pr_state":"open","pr_sha":"e372672579dc431b61fb58662ab3b2920b0e19e3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T15:18:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027402016820563968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T15:14:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b4b57bd41c5fee9c5a0bd13f30fad2e92ef53a01e04cbdd56dd3b322bc607b6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027402016820563968","occurred_at":"2026-02-27T15:14:42Z","failed_at":"provision","signature_id":"859f30b893f36a25e3aaabc11a5a3aa9969173a8d65dab34a2930960cfde3471","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T16:03:31.730Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T16:04:09.657Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.6Vrz4dROcC/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-jt2ikvc8/pipeline@sha256:02bf8b32699eb8194f3d4b6ddff936f87443e59186ac7ae065e9fe265e59ec7b to arohcpsvcdev.azurecr.io/ci-op-jt2ikvc8/pipeline:02bf8b32699eb8194f3d4b6ddff936f87443e59186ac7ae065e9fe265e59ec7b.\\nThe image will still be available under it's original digest sha256:02bf8b32699eb8194f3d4b6ddff936f87443e59186ac7ae065e9fe265e59ec7b in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027401328170373120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T15:11:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027400652421861376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T15:09:17Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e58bfa48c90979c2d98099bfd5d596ad0ca5cbacfc82171fc32a23b6652cdf0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027400652421861376","occurred_at":"2026-02-27T15:09:17Z","failed_at":"provision","signature_id":"e82a2a091efed186c854a8e3c740486092df9933dda65c377be60971953099ba","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T15:58:16.388Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T15:59:13.384Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.EYvDS2Zkoz/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-19ypnm6p/pipeline@sha256:00558cd649b38d3da0a0acc6e434ab220c36a802847e49bb8b45a5eb0893280b to arohcpsvcdev.azurecr.io/ci-op-19ypnm6p/pipeline:00558cd649b38d3da0a0acc6e434ab220c36a802847e49bb8b45a5eb0893280b.\\nThe image will still be available under it's original digest sha256:00558cd649b38d3da0a0acc6e434ab220c36a802847e49bb8b45a5eb0893280b in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027393914608029696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4241,"pr_state":"closed","pr_sha":"e356b899a0ad4235898c937e7414f6d1ad13ec5f","final_merged_sha":"f0eb454d49936e73b94a7710cef63a611155b521","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T14:42:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027381736224067584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3906,"pr_state":"closed","pr_sha":"4b0d70582dcc4f1b0a49ea8497ecc7d3bd5edf5c","final_merged_sha":"ab146996e5da6d3f4df071c1b9fd2ac214488fd6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T13:54:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fb0a9ca4099cdba14651fb611c0148d78e1bd647347a40eac988199b4dee06d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3906/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027381736224067584","occurred_at":"2026-02-27T13:54:07Z","failed_at":"e2e","signature_id":"2d2c80c357542ea2284d4749c23e8b7c809f636f03a5601d1bf6c65bf96b6d5b","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:147]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000884c00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-p25pd9\" to finish creating: GET https://rp.j4067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\",\n      \"name\": \"5bc4daeb-919d-4656-9e0b-89649609fd56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T15:12:27.639241982Z\",\n      \"endTime\": \"2026-02-27T15:14:18.918226609Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev/tokens/hc-ocp-pull-2onmcmpiplslohukepj06ujm7007gtnu\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: RegistryStatusConflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"RegistryStatusConflict\\\",\\n    \\\"message\\\": \\\"Cannot perform the operation as registry arohcpocpdev is in Updating status.\\\"\\n  },\\n  \\\"status\\\": \\\"Failed\\\"\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-p25pd9\\\" to finish creating: GET https://rp.j4067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\\\",\\n  \\\"name\\\": \\\"5bc4daeb-919d-4656-9e0b-89649609fd56\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T15:12:27.639241982Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T15:14:18.918226609Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev/tokens/hc-ocp-pull-2onmcmpiplslohukepj06ujm7007gtnu\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 400: 400 Bad Request\\\\nERROR CODE: RegistryStatusConflict\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"RegistryStatusConflict\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Cannot perform the operation as registry arohcpocpdev is in Updating status.\\\\\\\"\\\\n  },\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\"\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001138930\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 200,\n            RawResponse: {\n                Status: \"200 OK\",\n                StatusCode: 200,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"1168\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"e51a5e45-61c6-428f-a18e-a1f1e72db4fe\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 27 Feb 2026 15:14:28 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000884a80\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 53, 98, 99, 52, 100, 97, 101, 98, 45, 57, 49, 57, 100, 45, 52, 54, 53, 54, 45, 57, 101, 48, 98, 45, 56, 57, 54, 52, 57, 54, 48, 57, 102, 100, 53, 54, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 53, 98, 99, 52, 100, 97, 101, 98, 45, 57, 49, 57, 100, 45, 52, 54, 53, 54, 45, 57, 101, 48, 98, 45, 56, 57, 54, 52, 57, 54, 48, 57, 102, 100, 53, 54, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:147]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000884c00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-p25pd9\" to finish creating: GET https://rp.j4067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\",\n      \"name\": \"5bc4daeb-919d-4656-9e0b-89649609fd56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T15:12:27.639241982Z\",\n      \"endTime\": \"2026-02-27T15:14:18.918226609Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev/tokens/hc-ocp-pull-2onmcmpiplslohukepj06ujm7007gtnu\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: RegistryStatusConflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"RegistryStatusConflict\\\",\\n    \\\"message\\\": \\\"Cannot perform the operation as registry arohcpocpdev is in Updating status.\\\"\\n  },\\n  \\\"status\\\": \\\"Failed\\\"\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-p25pd9\\\" to finish creating: GET https://rp.j4067584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5bc4daeb-919d-4656-9e0b-89649609fd56\\\",\\n  \\\"name\\\": \\\"5bc4daeb-919d-4656-9e0b-89649609fd56\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T15:12:27.639241982Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T15:14:18.918226609Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev/tokens/hc-ocp-pull-2onmcmpiplslohukepj06ujm7007gtnu\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 400: 400 Bad Request\\\\nERROR CODE: RegistryStatusConflict\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"RegistryStatusConflict\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Cannot perform the operation as registry arohcpocpdev is in Updating status.\\\\\\\"\\\\n  },\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\"\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001138930\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 200,\n            RawResponse: {\n                Status: \"200 OK\",\n                StatusCode: 200,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Length\": [\"1168\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"e51a5e45-61c6-428f-a18e-a1f1e72db4fe\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 27 Feb 2026 15:14:28 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000884a80\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 53, 98, 99, 52, 100, 97, 101, 98, 45, 57, 49, 57, 100, 45, 52, 54, 53, 54, 45, 57, 101, 48, 98, 45, 56, 57, 54, 52, 57, 54, 48, 57, 102, 100, 53, 54, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 53, 98, 99, 52, 100, 97, 101, 98, 45, 57, 49, 57, 100, 45, 52, 54, 53, 54, 45, 57, 101, 48, 98, 45, 56, 57, 54, 52, 57, 54, 48, 57, 102, 100, 53, 54, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6163c1bf653b9f26ccf8bd31739b6688f84a7514a500eaca1ba54d2bef6f66d9","failure_pattern":"ERROR CODE: InternalServerError; detail code RegistryStatusConflict; detail message Cannot perform the operation as registry arohcpocpdev is in Updating status.; provider Microsoft.ContainerRegistry","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6163c1bf653b9f26ccf8bd31739b6688f84a7514a500eaca1ba54d2bef6f66d9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027366800315912192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3635,"pr_state":"closed","pr_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","final_merged_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T12:54:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027362800375959552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4117,"pr_state":"closed","pr_sha":"010fc6bfe121354ddf7e9de2a30f158e31d20617","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T12:38:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027361880825139200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4033,"pr_state":"closed","pr_sha":"f91614d0a917a65b393ce6f085de61f6fe832a81","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T12:35:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027361308206174208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3925,"pr_state":"open","pr_sha":"f9528e43607745096049cb9083bb5cb05d4898b5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T12:32:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4245/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027357021887533056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4245,"pr_state":"closed","pr_sha":"e183d0c0d3a8511f12ca1866b2dc836f32c58885","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T12:15:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027352343057534976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T11:57:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027352301944967168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4119,"pr_state":"closed","pr_sha":"8700ec5303d9646d72fd5761c2ccc9d31b35c5d0","final_merged_sha":"8700ec5303d9646d72fd5761c2ccc9d31b35c5d0","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T11:57:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027346219193339904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4244,"pr_state":"closed","pr_sha":"e0772ffca7ec26225cab1b0201d5162b797c2946","final_merged_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T11:32:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8baea493d1fe722577b1497f45c23122403325a72e86a2cba2690c5f1364e9b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027346219193339904","occurred_at":"2026-02-27T11:32:59Z","failed_at":"provision","signature_id":"369087d42aa89ff61e03263ca89fb4cad20785e17b53b6a4be92bb839d75dc17","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-27T12:19:48.936Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:19:49.344Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj3339904.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj3339904.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:dac1de9b-6368-4f61-a812-3665d74e215f databaseName:arohcpprow-rp-j3339904 databaseUrl:https://arohcpprow-rp-j3339904.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-9z1t4qjx/pipeline@sha256:2e1d98693a96033a7a46237aaff2214847d3d05b51fbbe771b4d4399649c563f replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-9z1t4qjx/pipeline:pull workloadIdentityClientId:26f97ceb-7566-4bab-a70c-1636d00a5f28 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:dac1de9b-6368-4f61-a812-3665d74e215f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:19:49.344Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:49.610Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:19:49.931Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:19:49.931Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:50.045Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:50.335Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"backend/templates/backend.deployment.yaml:68:25\\n  executing \\\"backend/templates/backend.deployment.yaml\\\" at \u003c.Values.miMock.certName\u003e:\\n    nil pointer evaluating interface {}.certName\"\ntime=2026-02-27T12:19:50.335Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:50.335Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:50.449Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"failed to get status for release aro-hcp-backend-dev: release: not found\"\ntime=2026-02-27T12:19:50.449Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:50.450Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: backend/templates/backend.deployment.yaml:68:25\\n  executing \\\"backend/templates/backend.deployment.yaml\\\" at \u003c.Values.miMock.certName\u003e:\\n    nil pointer evaluating interface {}.certName\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68d91604f3ad122a1d62133a42862f285cc314b8eee8ff4c3bfc85746defdd22","failure_pattern":"backend/templates/backend.deployment.yaml:68:25 executing \"backend/templates/backend.deployment.yaml\" at \u003c.Values.miMock.certName\u003e: nil pointer evaluating interface {}.certName","search_query":"time=2026-02-27T12:19:50.449Z level=ERROR msg=\"Failed to capture diagnostics for Helm release\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"failed to get status for release aro-h"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["68d91604f3ad122a1d62133a42862f285cc314b8eee8ff4c3bfc85746defdd22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4243/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027345665838813184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4243,"pr_state":"closed","pr_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","final_merged_sha":"33bde46988c6198e4a4c0a92b68af6e5e393d693","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T11:30:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027345038408683520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4242,"pr_state":"closed","pr_sha":"6cb3f1f9a4d2fd5d87f0beb3240e04cd94001c05","final_merged_sha":"e19daeb2d1145d00e1e3511c0cf61870b2446f97","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T11:28:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f37749b733f7d1001e4380243952a62c6addc767120054c8291e67fbbc933eb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027345038408683520","occurred_at":"2026-02-27T11:28:18Z","failed_at":"provision","signature_id":"53dd10b300bd9b13cee9bf37de252832790e98ec1894ff3c0e09ce778f8f7e10","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-27T12:13:26.602Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:13:27.070Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8683520.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8683520.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 databaseName:arohcpprow-rp-j8683520 databaseUrl:https://arohcpprow-rp-j8683520.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0m9v6ls2/pipeline@sha256:b3b4edc212ef944e28c86d668b5ba11e4650366eaf06fcc92635da54a366acb4 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0m9v6ls2/pipeline:pull workloadIdentityClientId:7ec5944e-14d7-4606-8783-e1cfc6bf6032 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:13:27.070Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:13:27.135Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:13:27.257Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:13:27.257Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:13:27.377Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:32.182Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-27T12:18:32.183Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:32.183Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:36.375Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8683520.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8683520.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 databaseName:arohcpprow-rp-j8683520 databaseUrl:https://arohcpprow-rp-j8683520.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0m9v6ls2/pipeline@sha256:b3b4edc212ef944e28c86d668b5ba11e4650366eaf06fcc92635da54a366acb4 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0m9v6ls2/pipeline:pull workloadIdentityClientId:7ec5944e-14d7-4606-8783-e1cfc6bf6032 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:18:36.377Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQW/aQBCF7/4VI59A8kaJIyUVVQ407alKhZqeUvWw3n2FDesda3YNIuLHV8alWAgars/fm3nP2sk8Egkit2IQ6YGsTjrpymPE1StM+uqCncQkLswL6qVvusY/CSuEnRIbbfby+GdGRJRPjcxa7z+5YF2Y5wXllTZLBKua1ntVHXQtrBamyYveOANk2qYFQnJGJ8dhAKn9kBpJnIkn/M+QlTOYGsNtSIPFJ9jv7NHJHtpCFDzM0cIheaLL+8bPaDxvavRRjmqc4LvqLO5t13zG3pnNjvKe1/vW/7E/wwjSTHjlLOTR6xiHeWP/c9QSG7XSrU/nf+ATB5dYLkv9yOG3mz/pZrhMv7UCJW1IroYyO+RS71n6b7oL30T262O2bCtIQEL80j3YmG1pvYCAulgx6bqhCmkNBBpZndDJo7y8Lu/UdanK+x835eTmdlLev+Rjurqis8yHye3dSz4eZ1t6ZRdo6YJ9cCFABmfGgQ7XNTyr43vKttQId18PQQtq2PbwZUMK8jwvqEaMet6NZLEQqjaD8hbR/AkAAP//x9KwYAwEAAA=\"\ntime=2026-02-27T12:18:36.380Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:36.380Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-27T12:19:36.380Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:19:37.227Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8683520.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8683520.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 databaseName:arohcpprow-rp-j8683520 databaseUrl:https://arohcpprow-rp-j8683520.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0m9v6ls2/pipeline@sha256:b3b4edc212ef944e28c86d668b5ba11e4650366eaf06fcc92635da54a366acb4 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0m9v6ls2/pipeline:pull workloadIdentityClientId:7ec5944e-14d7-4606-8783-e1cfc6bf6032 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:19:37.227Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:37.676Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:19:37.867Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:37.867Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:38.042Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-02-27T12:19:39.011Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:39.011Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-02-27T12:19:39.368Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-02-27T12:19:39.739Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-02-27T12:19:39.866Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-02-27T12:19:40.400Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-02-27T12:19:40.761Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:19:41.079Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:19:41.448Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-02-27T12:19:41.779Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-02-27T12:19:42.142Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:19:42.565Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-02-27T12:19:42.904Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:19:43.244Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:19:43.244Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:19:43.367Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-02-27T12:24:46.353Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:24:46.353Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:24:46.353Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:24:50.117Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8683520.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8683520.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 databaseName:arohcpprow-rp-j8683520 databaseUrl:https://arohcpprow-rp-j8683520.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0m9v6ls2/pipeline@sha256:b3b4edc212ef944e28c86d668b5ba11e4650366eaf06fcc92635da54a366acb4 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0m9v6ls2/pipeline:pull workloadIdentityClientId:7ec5944e-14d7-4606-8783-e1cfc6bf6032 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:24:50.119Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5STT2/TQBDF7/4UI58SyVulpoAI6iEUTqgoAk5FHNa7j2Sb9Y41u06UKh8eOVGIFTm0vT7/3vx53sk8Egkit2IQ6ZasTjrpymPE1SNM+uqCncYkLiwKOkjfdI1/EtYIeyU22hzl8a+MiCifGZm33n9ywbqwyAvKK21WCFY1rfeqOulaWC1NkxcH4xyQWZuWCMkZnRyHHqSORWokcSYO+H/ACNJceO0s5M7rGPvdI2TtDNQKW7XWrU+DJfbMPQeXWAbaX/bMjOE2pF7HAfY7ewwE46EtRMHDnK191uR1gXRhsrinfZZz9s5s95T3vDna/rPYHYc/bnGvm/6o+qkVKGlDcjWU2SMv9V6ku1g6+fkcPqPxvK0R0jM/J/v9MVu1FSQgIX7pHmzMdrRZQkDd7DHpuqEKaQMEGlmd0MmjvJyU79SkVOX7n9fl9PrD9ObNQz6mqyu6xJQ307eTh3w8znb0yC7QygV760KA9M6MA52uq39W5/eU7agR7r6eBi2oYXuAX1akIM+LgmrEqBddSRYLoWrbW94imr8BAAD//7rwSicMBAAA\"\ntime=2026-02-27T12:24:50.121Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:24:50.121Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:25:50.121Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:25:51.025Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8683520.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8683520.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 databaseName:arohcpprow-rp-j8683520 databaseUrl:https://arohcpprow-rp-j8683520.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0m9v6ls2/pipeline@sha256:b3b4edc212ef944e28c86d668b5ba11e4650366eaf06fcc92635da54a366acb4 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0m9v6ls2/pipeline:pull workloadIdentityClientId:7ec5944e-14d7-4606-8783-e1cfc6bf6032 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:25:51.025Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:25:51.343Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:25:51.639Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:25:51.639Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:25:51.817Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-02-27T12:25:52.845Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:25:52.845Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-02-27T12:25:53.213Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-02-27T12:25:53.564Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-02-27T12:25:53.694Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-02-27T12:25:54.192Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-02-27T12:25:54.508Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:25:54.800Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:25:55.181Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-02-27T12:25:55.498Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-02-27T12:25:55.915Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:25:56.282Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-02-27T12:25:56.588Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:25:56.892Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:25:56.892Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:25:57.074Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-02-27T12:31:00.160Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:31:00.160Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:31:00.160Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:31:04.406Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8683520.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8683520.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 databaseName:arohcpprow-rp-j8683520 databaseUrl:https://arohcpprow-rp-j8683520.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0m9v6ls2/pipeline@sha256:b3b4edc212ef944e28c86d668b5ba11e4650366eaf06fcc92635da54a366acb4 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0m9v6ls2/pipeline:pull workloadIdentityClientId:7ec5944e-14d7-4606-8783-e1cfc6bf6032 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:83608b2d-35bd-4b05-a8dd-aea3a3e8abf2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:31:04.407Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTT2/aQBDF7/4UI5+C5I2I26QSVQ407alKhdqeUvWw3n2FDetda3YWRMSHr0xCsRCoXN/+3ps/9hQeQowUMxskuierRYtuPK5i8wwjX12wkyTswryiV+mbbvFPwgphp6ROm708+lUQEZU/YBgy47hyFvzgdUplRWWjzRLBqgReOQO1xEatdPbSP2qOamG6stpH7JjHGJxEHgDqLeW8Z2pMzEEGFU+wDzH8cfNH3Q0b0y+ZoTgHcS2U2SGXes/S36NHL3toC1bwMOJiOEN+csG6MB8m/9/4NviJLakWws6kE6bP6HzctAhy2XZnAE+zLBDEGX3UyQX1poZn2fsTA3bZe9Uc9GNjlkVk97KrOYvemc2O8j6u9/VO9V38/lgscwMOEKQv/Q+bii2tF2BQ/4WT6LajBrIGAl1ZLejlq7Ie13dqXKv6w8+belLfTm7vnsoRXV/TOebdzWT8/qkcjYotPUcXaOmCvXchgAdnFgMdrmt4Vsf3VGyp49i/HhqtqIv2Fb4spCIf5xW1SEnP+8jIFkzNZjC8RTJ/AwAA///wEZfkDAQAAA==\"\ntime=2026-02-27T12:31:04.411Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:31:04.411Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:32:04.416Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027343551058153472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4241,"pr_state":"closed","pr_sha":"d07d3f74e66ac27df4e3bab88342b25bc38d05fc","final_merged_sha":"f0eb454d49936e73b94a7710cef63a611155b521","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T11:22:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341673876426752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3820,"pr_state":"closed","pr_sha":"ddfe2c8cc637a5031ec57809e55c715309a8336d","final_merged_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T11:14:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ccc45376783b8b5e54e3a217a30b949f9fabd23137a1d926194e9384005e33c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341673876426752","occurred_at":"2026-02-27T11:14:55Z","failed_at":"provision","signature_id":"82a8d7bc439b3aaf73237c6ba3957be7fb6175835a5863252a40d85cdbec9dbc","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-27T12:06:02.439Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:06:03.044Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6426752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6426752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:95227438-b9b0-4364-b936-90a72f85e701 databaseName:arohcpprow-rp-j6426752 databaseUrl:https://arohcpprow-rp-j6426752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-ymccrjwx/pipeline@sha256:4d7ed282c8f0d436517235e97cdbb1f663664ae371bafc1ec74714fcfcabd366 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-ymccrjwx/pipeline:pull workloadIdentityClientId:075cde89-c30e-4e4d-a8c9-d0d39d246bad workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:95227438-b9b0-4364-b936-90a72f85e701 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:06:03.044Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:06:03.105Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:06:03.222Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:06:03.222Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:06:03.336Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:11:07.242Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-27T12:11:07.242Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:11:07.242Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:11:11.290Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6426752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6426752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:95227438-b9b0-4364-b936-90a72f85e701 databaseName:arohcpprow-rp-j6426752 databaseUrl:https://arohcpprow-rp-j6426752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-ymccrjwx/pipeline@sha256:4d7ed282c8f0d436517235e97cdbb1f663664ae371bafc1ec74714fcfcabd366 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-ymccrjwx/pipeline:pull workloadIdentityClientId:075cde89-c30e-4e4d-a8c9-d0d39d246bad workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:95227438-b9b0-4364-b936-90a72f85e701 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:11:11.292Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5STQWvbQBCF7/oVg04xaIOtQgouOaRpTyXFND2l9LDafZU3Xu2I2ZWNg398kR3Xwshten1635s3YifzSCSI3IlBpFuyOumkK48rrp5h0hcX7DwmcaEu6CB91Q3+SFgj7JXYanOUJz8yIqL8nsMvVz/oNi8or7RZIVilXzqBki4k10CZvaX/roXV0rR58Rf2ovvOyKLz/qML1oV6iLSd96o66edgl5Ys7kUnx2HB3pnt3uU9b1SDJM5E9Zo1gi8A6SMQkjP7jIHpyB1zRvhHyNoZPHBwiWWEHWG+sUcve2gLUfAwZ4OHzpFf8m/wtdb/7fIJredtg5DetscjjCAthNfOQu69jnHYMh46qBW2aq07ny73vDOGu8PYkXHZzw/ZqqsgAQnxc/9gY7ajzRIC6h9hTLppqULaAIGurE7o5au8nJY3alqq8v33WTmf3syn757yCV1f0yXPbDafzZ7yySTb0TO7QCsX7K0LATI4Mw50uq7hWZ3fU7ajVrj/eipaUMv2YH5bSEGe64IaxKjrPpLFQqjaDpa3iOZ3AAAA//+AUkQkDAQAAA==\"\ntime=2026-02-27T12:11:11.296Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:11:11.296Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-27T12:12:11.301Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:12:11.871Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6426752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6426752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:95227438-b9b0-4364-b936-90a72f85e701 databaseName:arohcpprow-rp-j6426752 databaseUrl:https://arohcpprow-rp-j6426752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-ymccrjwx/pipeline@sha256:4d7ed282c8f0d436517235e97cdbb1f663664ae371bafc1ec74714fcfcabd366 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-ymccrjwx/pipeline:pull workloadIdentityClientId:075cde89-c30e-4e4d-a8c9-d0d39d246bad workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:95227438-b9b0-4364-b936-90a72f85e701 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:12:11.871Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:12:12.234Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:12:12.526Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:12:12.526Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:12:12.756Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-02-27T12:12:13.745Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:12:13.745Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-02-27T12:12:14.143Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-02-27T12:12:14.529Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-02-27T12:12:14.695Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-02-27T12:12:15.316Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-02-27T12:12:15.631Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:12:15.965Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:12:16.284Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-02-27T12:12:16.599Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-02-27T12:12:16.968Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:12:17.384Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-02-27T12:12:17.701Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:12:18.251Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:12:18.251Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:12:18.382Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-02-27T12:17:21.603Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:17:21.604Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:17:21.604Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:17:25.432Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6426752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6426752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:95227438-b9b0-4364-b936-90a72f85e701 databaseName:arohcpprow-rp-j6426752 databaseUrl:https://arohcpprow-rp-j6426752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-ymccrjwx/pipeline@sha256:4d7ed282c8f0d436517235e97cdbb1f663664ae371bafc1ec74714fcfcabd366 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-ymccrjwx/pipeline:pull workloadIdentityClientId:075cde89-c30e-4e4d-a8c9-d0d39d246bad workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:95227438-b9b0-4364-b936-90a72f85e701 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:17:25.433Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5STT2/TQBDF7/4UI58ayVsVS1AU1EMonFBRBJyKOKx3H8426x1rdp0oVT482oQQEzW0SD49/+bP25kpPBIJIg9iEOmGrE466cbjgpsHmPTJBTuNSVxoK9pLn3WHPxJWCDsl9toc5Mn3goio/ApZOYM7Di6xlBWVWlgtTK8abZYIdiSV1V8xM2N4CCkD59lbDj9de6f7Eab04yBQMoTkOiizQ14ae5b+wh5Z9tAWouBhkuPwBDkzMh+8f++CdaEdJ+8H71Vz1E8C54DMhrRASM7ok+yHB1MdkjgTz7T4RNXnO/794v9X7QN6z5sO+xE9P9bsjMU97ozN2Tuz2VHe8/pQ5p9bYQRpLrxyFnLrdYxjk3FvQS2xUSs9+HFTZfHjXbEcGkhAQvyYFzYWW1ovIKC8JDHprqcGaQ0EurA6IcsXZX1Vv1FXtaqvv72qp/l7e19O6PKSzjLX0/r1fTmZFFt6YBdo6YK9cSFARmfGgY7XNT6r03sqttQL57/HRivq2e7hlyWpyHNbUYcYdZtTslgINZuReYtofgUAAP//ayOzsQwEAAA=\"\ntime=2026-02-27T12:17:25.434Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:17:25.435Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:18:25.438Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T12:18:26.395Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6426752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6426752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:95227438-b9b0-4364-b936-90a72f85e701 databaseName:arohcpprow-rp-j6426752 databaseUrl:https://arohcpprow-rp-j6426752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-ymccrjwx/pipeline@sha256:4d7ed282c8f0d436517235e97cdbb1f663664ae371bafc1ec74714fcfcabd366 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-ymccrjwx/pipeline:pull workloadIdentityClientId:075cde89-c30e-4e4d-a8c9-d0d39d246bad workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:95227438-b9b0-4364-b936-90a72f85e701 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:18:26.395Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:26.930Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-27T12:18:27.148Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:27.148Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:27.445Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-02-27T12:18:28.460Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:28.460Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ServiceAccount\" namespace=aro-hcp name=backend\ntime=2026-02-27T12:18:28.799Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-azure-runtime-config\ntime=2026-02-27T12:18:29.164Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=ConfigMap\" namespace=aro-hcp name=backend-config\ntime=2026-02-27T12:18:29.359Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=Role\" namespace=aro-hcp name=leader-election\ntime=2026-02-27T12:18:29.956Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"rbac.authorization.k8s.io/v1, Kind=RoleBinding\" namespace=aro-hcp name=backend-leader-election\ntime=2026-02-27T12:18:30.279Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"/v1, Kind=Service\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:18:30.612Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"apps/v1, Kind=Deployment\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:18:31.127Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"acrpull.microsoft.com/v1beta2, Kind=AcrPullBinding\" namespace=aro-hcp name=backend-pull-binding\ntime=2026-02-27T12:18:31.489Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1, Kind=AuthorizationPolicy\" namespace=aro-hcp name=allow-metrics-backend\ntime=2026-02-27T12:18:31.907Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"security.istio.io/v1beta1, Kind=PeerAuthentication\" namespace=aro-hcp name=aro-hcp-backend-metrics\ntime=2026-02-27T12:18:32.341Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"secrets-store.csi.x-k8s.io/v1, Kind=SecretProviderClass\" namespace=aro-hcp name=backend-service-key-vault\ntime=2026-02-27T12:18:32.650Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=aro-hcp name=aro-hcp-backend\ntime=2026-02-27T12:18:33.057Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T12:18:33.057Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:18:33.226Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-02-27T12:23:36.221Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:23:36.221Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:23:36.221Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:23:40.897Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj6426752.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj6426752.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:95227438-b9b0-4364-b936-90a72f85e701 databaseName:arohcpprow-rp-j6426752 databaseUrl:https://arohcpprow-rp-j6426752.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-ymccrjwx/pipeline@sha256:4d7ed282c8f0d436517235e97cdbb1f663664ae371bafc1ec74714fcfcabd366 replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-ymccrjwx/pipeline:pull workloadIdentityClientId:075cde89-c30e-4e4d-a8c9-d0d39d246bad workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:95227438-b9b0-4364-b936-90a72f85e701 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-27T12:23:40.898Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5RTz2vbMBi9+6/48KkBq6TO2EZGD1m30+gI604dO8jSW6JG1mc+yQkp+eOHHbKYEtPu+vR++Ml6mUciQeRWDCLdktVJJ115XHH1BJO+uWDnMYkLq4KO0Hdd4x+ELUKPxEabEzz5lRER5Qsjy9b7zy5YF1Z5QXmlzQbBqqb1XlVnXAurtWny4ih8gGydwcIYbkMaCMe5gxN1SqmRxJl4QbRo05rFPevkOCzZO7PvWd7z7iRT45lLQDoLhORM7/F/8Q8wgrQU3joLufM6xuHtxGMltcFebXXr03jtew4usVyIv6C54/DHre51MwzTz61ASRuSq6FMT3mrdpT9g33/Rzy0hSh4mBe3NGReeCCvC7+g8byvEdIr5bPfn7JNW0ECEuLX7sHG7EC7NQTUlY5J1w1VSDsg0JXVCR18lZfT8r2alqr88POmnN98nM9mj/mErq9pjFPO5u+mj/lkkh3oiV2gjQv21oUAGcyMA53XNZzVyz1lB2qEu9PzhxbUsD2S32ZSkOdVQTVi1KvOksVCqNoPyltE8zcAAP//IOGX5gwEAAA=\"\ntime=2026-02-27T12:23:40.900Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-27T12:23:40.901Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-27T12:24:40.908Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4240,"pr_state":"closed","pr_sha":"b192f65a2fc144f32c9f4adc2b9fad41d1f08321","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T11:13:24Z"},"failed_at":["e2e"],"failed_test_count":20,"occurrences":[{"row_id":"09f242a4907ac71eeecafe4bfddad7f171a9e7b777ea00cad9eb31b7995e8278","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"2779a1a964e0356c5d392fa66d267845c58575ef5e223771ff3e069fa1e70be8","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007b73a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-v9wcdw\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\",\n      \"name\": \"cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:57.846365549Z\",\n      \"endTime\": \"2026-02-27T12:16:32.507806163Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-v9wcdw\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"name\\\": \\\"cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.846365549Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.507806163Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007b7340\u003e{\n            msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-v9wcdw\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"name\\\": \\\"cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.846365549Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.507806163Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e79e90\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007b73a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-v9wcdw\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\",\n      \"name\": \"cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:57.846365549Z\",\n      \"endTime\": \"2026-02-27T12:16:32.507806163Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-v9wcdw\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"name\\\": \\\"cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.846365549Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.507806163Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007b7340\u003e{\n            msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-v9wcdw\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"name\\\": \\\"cce67aa8-a8b5-4fc5-b9e5-5d330bf58c32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.846365549Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.507806163Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v9wcdw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e79e90\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22b50ea2e49b9f8636a928022d9c6fb867bddf916bb5e846d7ed255b2bb5febf","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"1b7f382747554c7d29e1f4ddea3c25a079bf260697b691fd432adbf347ded41b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"1fc942d847abcdc2fd2ca0bc377c630930cee65cc62ee7518a7413020d355652","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088b060\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-m48g8f\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\",\n      \"name\": \"97864547-a6ee-47f0-afbf-d12be3a4a990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:38.539120748Z\",\n      \"endTime\": \"2026-02-27T12:16:32.597846564Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-m48g8f\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"name\\\": \\\"97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.539120748Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597846564Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088b040\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-m48g8f\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"name\\\": \\\"97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.539120748Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597846564Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000f507e0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                Status...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088b060\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-m48g8f\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\",\n      \"name\": \"97864547-a6ee-47f0-afbf-d12be3a4a990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:38.539120748Z\",\n      \"endTime\": \"2026-02-27T12:16:32.597846564Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-m48g8f\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"name\\\": \\\"97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.539120748Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597846564Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088b040\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-m48g8f\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"name\\\": \\\"97864547-a6ee-47f0-afbf-d12be3a4a990\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.539120748Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597846564Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-m48g8f/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000f507e0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                Status...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"298c1e8b58b3eec0cad62cbc882f65922774d688ac5a805c14dc5324c893cbd9","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"2e17a6ed928909c44ff22e47c541da01ba06402809c534ddd7e812773ca664a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"c4f1e539b6aa071ca631b854e88ec8c4cdcfb169f25cebbac086ec78cc9fa126","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001351360\u003e: \n    failed to create HCP cluster list-test-cluster-4vbdmp: failed waiting for cluster=\"list-test-cluster-4vbdmp\" in resourcegroup=\"cluster-listing-hrs9dt\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\",\n      \"name\": \"0d3f0a38-a251-4bea-be1e-7df5e63798a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:57.420631812Z\",\n      \"endTime\": \"2026-02-27T12:16:32.583975879Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster list-test-cluster-4vbdmp: failed waiting for cluster=\\\"list-test-cluster-4vbdmp\\\" in resourcegroup=\\\"cluster-listing-hrs9dt\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"name\\\": \\\"0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.420631812Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.583975879Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001351340\u003e{\n            msg: \"failed waiting for cluster=\\\"list-test-cluster-4vbdmp\\\" in resourcegroup=\\\"cluster-listing-hrs9dt\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"name\\\": \\\"0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.420631812Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.583975879Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010d0c60\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001351360\u003e: \n    failed to create HCP cluster list-test-cluster-4vbdmp: failed waiting for cluster=\"list-test-cluster-4vbdmp\" in resourcegroup=\"cluster-listing-hrs9dt\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\",\n      \"name\": \"0d3f0a38-a251-4bea-be1e-7df5e63798a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:57.420631812Z\",\n      \"endTime\": \"2026-02-27T12:16:32.583975879Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster list-test-cluster-4vbdmp: failed waiting for cluster=\\\"list-test-cluster-4vbdmp\\\" in resourcegroup=\\\"cluster-listing-hrs9dt\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"name\\\": \\\"0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.420631812Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.583975879Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001351340\u003e{\n            msg: \"failed waiting for cluster=\\\"list-test-cluster-4vbdmp\\\" in resourcegroup=\\\"cluster-listing-hrs9dt\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"name\\\": \\\"0d3f0a38-a251-4bea-be1e-7df5e63798a9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:57.420631812Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.583975879Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-hrs9dt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010d0c60\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22b50ea2e49b9f8636a928022d9c6fb867bddf916bb5e846d7ed255b2bb5febf","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"2f77b81e7d24d65cb10664f6d3ef43501184bb35db616a9f75ea6e054c8f5e3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"769e387209cb034f3aad45368f01f8f6059caeaa2320f0f8a57ff11d69eb3cc0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00011eac0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8tzqbc\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\",\n      \"name\": \"0d638fc4-cb24-4127-a7a4-0731ba807541\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:21:28.06672496Z\",\n      \"endTime\": \"2026-02-27T12:21:56.643508269Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8tzqbc\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"name\\\": \\\"0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:28.06672496Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.643508269Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00011eaa0\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8tzqbc\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"name\\\": \\\"0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:28.06672496Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.643508269Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012f0360\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n           ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00011eac0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8tzqbc\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\",\n      \"name\": \"0d638fc4-cb24-4127-a7a4-0731ba807541\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:21:28.06672496Z\",\n      \"endTime\": \"2026-02-27T12:21:56.643508269Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8tzqbc\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"name\\\": \\\"0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:28.06672496Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.643508269Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00011eaa0\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-8tzqbc\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"name\\\": \\\"0d638fc4-cb24-4127-a7a4-0731ba807541\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:28.06672496Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.643508269Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8tzqbc/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012f0360\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n           ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"838b49e8b50324bb3927086023d5f7c85bc454461244d8ef112fe4351847988d","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"3735c80ff39cc6d05aa116c670ae183658ee61726cbdc9244e4a11f2b88776af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"bede48cbdfae5767015b4ba13a9e2f90b8ee1dd4777b7f3bb9b579c45fb2cdbe","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000928840\u003e: \n    failed to create HCP cluster ea-listcdrnpw: failed waiting for cluster=\"ea-listcdrnpw\" in resourcegroup=\"ea-list-52d4d8\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\",\n      \"name\": \"26d223df-dc7e-413a-b6e8-42de1919ffc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:16:06.174126807Z\",\n      \"endTime\": \"2026-02-27T12:16:32.619765973Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listcdrnpw: failed waiting for cluster=\\\"ea-listcdrnpw\\\" in resourcegroup=\\\"ea-list-52d4d8\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"name\\\": \\\"26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:16:06.174126807Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.619765973Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000928820\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listcdrnpw\\\" in resourcegroup=\\\"ea-list-52d4d8\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"name\\\": \\\"26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:16:06.174126807Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.619765973Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ae52c0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2....\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000928840\u003e: \n    failed to create HCP cluster ea-listcdrnpw: failed waiting for cluster=\"ea-listcdrnpw\" in resourcegroup=\"ea-list-52d4d8\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\",\n      \"name\": \"26d223df-dc7e-413a-b6e8-42de1919ffc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:16:06.174126807Z\",\n      \"endTime\": \"2026-02-27T12:16:32.619765973Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listcdrnpw: failed waiting for cluster=\\\"ea-listcdrnpw\\\" in resourcegroup=\\\"ea-list-52d4d8\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"name\\\": \\\"26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:16:06.174126807Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.619765973Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000928820\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listcdrnpw\\\" in resourcegroup=\\\"ea-list-52d4d8\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"name\\\": \\\"26d223df-dc7e-413a-b6e8-42de1919ffc7\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:16:06.174126807Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.619765973Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-52d4d8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ae52c0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2....\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22b50ea2e49b9f8636a928022d9c6fb867bddf916bb5e846d7ed255b2bb5febf","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"382edc32883afffb2d85859072623d29a407668df42411574b809259eadbfd57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"ec70cca27125cacbc24aa16bfb43d54ef31c0c1bca339895afd88cbdbe2f0087","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0012367b0\u003e: \n    GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8d70e5d1-5631-49ed-a77a-a1966e31343e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8d70e5d1-5631-49ed-a77a-a1966e31343e\",\n      \"name\": \"8d70e5d1-5631-49ed-a77a-a1966e31343e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:16:46.504949762Z\",\n      \"endTime\": \"2026-02-27T12:17:38.713829946Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-rsttbd/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-rsttbd/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"MultipleErrorsOccurred\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Ms-Request-Id\": [\n                    \"4f565cbc-d068-4843-b1f9-5f5e940392c7\",\n                ],\n                \"Date\": [\n                    \"Fri, 27 Feb 2026 12:17:46 GMT\",\n                ],\n                \"Content-Length\": [\"1218\"],\n                \"X-Envoy-Upstream-Service-Time\": [\"12\"],\n                \"Server\": [\"istio-envoy\"],\n                \"Content-Type\": [\"application/json\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000a06240\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 101, 97, 115, 116, 117, 115, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 56, 100, 55, 48, 101, 53, 100, 49, 45, 53, 54, 51, 49, 45, 52, 57, 101, 100, 45, 97, 55, 55, 97, 45, 97, 49, 57, 54, 54, 101, 51, 49, 51, 52, 51, 101, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 56, 100, 55, 48, 101, 53, 100, 49, 45, 53, 54, 51, 49, 45, 52, 57, 101, 100, 45, 97, 55, 55, 97, 45, 97, 49, 57, 54, 54, 101, 51, 49, 51, 52, 51, 101, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 50, 45, 50, 55, 84, 49, 50, 58, 49, 54, 58, 52, 54, 46, 53, 48, 52, 57, 52, 57, 55, 54, 50, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 50, 45, 50, 55, 84, 49, 50, 58, 49, 55, 58, 51, 56, 46, 55, 49, 51, 56, 50, 57, 57, 52, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 109, 117, 108, 116, 105, 112, 108, 101, 32, 101, 114, 114, 111, 114, 115, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 103, 105, 111, 110, 32, 111, 102, 32, 116, 104, 101, 32, 115, 117, 98, 110, 101, 116, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 99, 117, 115, 116, 111, 109, 101, 114, 45, 114, 103, 45, 114, 115, 116, 116, 98, 100, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 78, 101, 116, 119, 111, 114, 107, 4...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0012367b0\u003e: \n    GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8d70e5d1-5631-49ed-a77a-a1966e31343e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8d70e5d1-5631-49ed-a77a-a1966e31343e\",\n      \"name\": \"8d70e5d1-5631-49ed-a77a-a1966e31343e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:16:46.504949762Z\",\n      \"endTime\": \"2026-02-27T12:17:38.713829946Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-rsttbd/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-rsttbd/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"MultipleErrorsOccurred\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Ms-Request-Id\": [\n                    \"4f565cbc-d068-4843-b1f9-5f5e940392c7\",\n                ],\n                \"Date\": [\n                    \"Fri, 27 Feb 2026 12:17:46 GMT\",\n                ],\n                \"Content-Length\": [\"1218\"],\n                \"X-Envoy-Upstream-Service-Time\": [\"12\"],\n                \"Server\": [\"istio-envoy\"],\n                \"Content-Type\": [\"application/json\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000a06240\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 101, 97, 115, 116, 117, 115, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 56, 100, 55, 48, 101, 53, 100, 49, 45, 53, 54, 51, 49, 45, 52, 57, 101, 100, 45, 97, 55, 55, 97, 45, 97, 49, 57, 54, 54, 101, 51, 49, 51, 52, 51, 101, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 56, 100, 55, 48, 101, 53, 100, 49, 45, 53, 54, 51, 49, 45, 52, 57, 101, 100, 45, 97, 55, 55, 97, 45, 97, 49, 57, 54, 54, 101, 51, 49, 51, 52, 51, 101, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 50, 45, 50, 55, 84, 49, 50, 58, 49, 54, 58, 52, 54, 46, 53, 48, 52, 57, 52, 57, 55, 54, 50, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 50, 45, 50, 55, 84, 49, 50, 58, 49, 55, 58, 51, 56, 46, 55, 49, 51, 56, 50, 57, 57, 52, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 109, 117, 108, 116, 105, 112, 108, 101, 32, 101, 114, 114, 111, 114, 115, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 103, 105, 111, 110, 32, 111, 102, 32, 116, 104, 101, 32, 115, 117, 98, 110, 101, 116, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 99, 117, 115, 116, 111, 109, 101, 114, 45, 114, 103, 45, 114, 115, 116, 116, 98, 100, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 78, 101, 116, 119, 111, 114, 107, 4...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a169dbf582799690cc6c7941807a00ed4f37dceb52da9fd969e592d60674934b","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"3a39f1e279603f95964bbee70b69f401e9e4552708e569b8234fdc0e50283f49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"6eeac7507ffc098188e8161dfb8b1d8d52c09ccd74babfefc2dced33725df9b4","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:147]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001090900\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5hshbq\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\",\n      \"name\": \"2f93981c-cee6-4f14-bde6-e8920d165a6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:37.255025972Z\",\n      \"endTime\": \"2026-02-27T12:16:32.55881838Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19/subnets/customer-vnet-subnet-4-19' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name-4-19' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5hshbq\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\\\",\\n  \\\"name\\\": \\\"2f93981c-cee6-4f14-bde6-e8920d165a6b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:37.255025972Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.55881838Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19/subnets/customer-vnet-subnet-4-19' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name-4-19' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0011712f0\u003e{\n            ErrorCode: \"MultipleErrorsOccurred\",\n            StatusCode: 200,\n            RawResponse: {\n                Status: \"200 OK\",\n                StatusCode: 200,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"55\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"11a41819-0448-464f-a96b-45ed721ec1ed\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 27 Feb 2026 12:16:37 GMT\",\n                    ],\n                    \"Content-Length\": [\"1255\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0010907c0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 101, 97, 115, 116, 117, 115, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 50, 102, 57, 51, 57, 56, 49, 99, 45, 99, 101, 101, 54, 45, 52, 102, 49, 52, 45, 98, 100, 101, 54, 45, 101, 56, 57, 50, 48, 100, 49, 54, 53, 97, 54, 98, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 50, 102, 57, 51, 57, 56, 49, 99, 45, 99, 101, 101, 54, 45, 52, 102, 49, 52, 45, 98, 100, 101, 54, 45, 101, 56, 57, 50, 48, 100, 49, 54, 53, 97, 54, 98, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:147]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001090900\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5hshbq\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\",\n      \"name\": \"2f93981c-cee6-4f14-bde6-e8920d165a6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:37.255025972Z\",\n      \"endTime\": \"2026-02-27T12:16:32.55881838Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19/subnets/customer-vnet-subnet-4-19' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name-4-19' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5hshbq\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/2f93981c-cee6-4f14-bde6-e8920d165a6b\\\",\\n  \\\"name\\\": \\\"2f93981c-cee6-4f14-bde6-e8920d165a6b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:37.255025972Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.55881838Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19/subnets/customer-vnet-subnet-4-19' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5hshbq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name-4-19' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0011712f0\u003e{\n            ErrorCode: \"MultipleErrorsOccurred\",\n            StatusCode: 200,\n            RawResponse: {\n                Status: \"200 OK\",\n                StatusCode: 200,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"55\"],\n                    \"Server\": [\"istio-envoy\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"11a41819-0448-464f-a96b-45ed721ec1ed\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 27 Feb 2026 12:16:37 GMT\",\n                    ],\n                    \"Content-Length\": [\"1255\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0010907c0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 101, 97, 115, 116, 117, 115, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 50, 102, 57, 51, 57, 56, 49, 99, 45, 99, 101, 101, 54, 45, 52, 102, 49, 52, 45, 98, 100, 101, 54, 45, 101, 56, 57, 50, 48, 100, 49, 54, 53, 97, 54, 98, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 50, 102, 57, 51, 57, 56, 49, 99, 45, 99, 101, 101, 54, 45, 52, 102, 49, 52, 45, 98, 100, 101, 54, 45, 101, 56, 57, 50, 48, 100, 49, 54, 53, 97, 54, 98, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 3...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cad7dda9eb8f2d31e02c5973f26c8b8c1d060284fa95a731ef1903c26c11ba9a","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name-4-19' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"3f72b48a0174fdfb08a9853b225cbe749b160afc0d7199dd846e8aa4b034bcc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"19281c8c992fcfc6de82d31e5f0d59cbbef563339ee8ec934d84c139298ea1e6","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009ac2a0\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-sg74dp\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\",\n      \"name\": \"df272308-e591-45ee-b1d0-df3721e63d32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:56.78769024Z\",\n      \"endTime\": \"2026-02-27T12:16:32.544626386Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sg74dp\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"name\\\": \\\"df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.78769024Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.544626386Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009ac260\u003e{\n            msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sg74dp\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"name\\\": \\\"df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.78769024Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.544626386Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0003909f0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                R...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009ac2a0\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-sg74dp\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\",\n      \"name\": \"df272308-e591-45ee-b1d0-df3721e63d32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:56.78769024Z\",\n      \"endTime\": \"2026-02-27T12:16:32.544626386Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sg74dp\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"name\\\": \\\"df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.78769024Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.544626386Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009ac260\u003e{\n            msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sg74dp\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"name\\\": \\\"df272308-e591-45ee-b1d0-df3721e63d32\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.78769024Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.544626386Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-sg74dp/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0003909f0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                R...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"838b49e8b50324bb3927086023d5f7c85bc454461244d8ef112fe4351847988d","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"4282fedecc93be596f6b688e5d33ffa5a307d6d3aa69019a86607fab2bf66dff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"d5a64bfce4856b1e5e86d012d672868c25f0730c4345b90c3248f3294080529d","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a77680\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-599qjn\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\",\n      \"name\": \"db0aeaf8-2626-4566-a8e3-e229f634094b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:36.262310999Z\",\n      \"endTime\": \"2026-02-27T12:16:32.316563551Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-599qjn\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"name\\\": \\\"db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:36.262310999Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.316563551Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a77640\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-599qjn\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"name\\\": \\\"db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:36.262310999Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.316563551Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010ee0c0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n         ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a77680\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-599qjn\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\",\n      \"name\": \"db0aeaf8-2626-4566-a8e3-e229f634094b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:36.262310999Z\",\n      \"endTime\": \"2026-02-27T12:16:32.316563551Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-599qjn\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"name\\\": \\\"db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:36.262310999Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.316563551Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a77640\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-599qjn\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"name\\\": \\\"db0aeaf8-2626-4566-a8e3-e229f634094b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:36.262310999Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.316563551Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-599qjn/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010ee0c0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n         ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"838b49e8b50324bb3927086023d5f7c85bc454461244d8ef112fe4351847988d","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"474b3004f40f58d1e33b3cde456aa5022c6e7aa57fd3aef3194af01d415e8998","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"8376d7e86ddb032c6555078e7bb5d89e1d89dda127af03c9c428f42cb5c83104","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088f1c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-vbxc8m\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\",\n      \"name\": \"352cf7af-ac41-406d-b917-fae6f83901e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:38.209863996Z\",\n      \"endTime\": \"2026-02-27T12:16:32.558303866Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-vbxc8m\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"name\\\": \\\"352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.209863996Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.558303866Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088f1a0\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-vbxc8m\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"name\\\": \\\"352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.209863996Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.558303866Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001799140\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088f1c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-vbxc8m\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\",\n      \"name\": \"352cf7af-ac41-406d-b917-fae6f83901e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:38.209863996Z\",\n      \"endTime\": \"2026-02-27T12:16:32.558303866Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-vbxc8m\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"name\\\": \\\"352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.209863996Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.558303866Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088f1a0\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-vbxc8m\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"name\\\": \\\"352cf7af-ac41-406d-b917-fae6f83901e8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:38.209863996Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.558303866Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-vbxc8m/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001799140\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22010894ce78de64ead3e990516c53be92bf6fabb33ebfeb05fa683edb8b6071","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"528e55d4f310d0007989a7d409a1844307930d5cd147777f368803fa0b111b44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"bd572afb7e3064a468caf16385dbe79d122974aefa86596667c9df9103035f1c","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000734ec0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-qds7wl\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\",\n      \"name\": \"fe40c452-f274-4427-8d6a-5d464acb5998\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:47.421017831Z\",\n      \"endTime\": \"2026-02-27T12:16:32.344686333Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qds7wl\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"name\\\": \\\"fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:47.421017831Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.344686333Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000734e60\u003e{\n            msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qds7wl\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"name\\\": \\\"fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:47.421017831Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.344686333Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001203890\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusC...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000734ec0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-qds7wl\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\",\n      \"name\": \"fe40c452-f274-4427-8d6a-5d464acb5998\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:47.421017831Z\",\n      \"endTime\": \"2026-02-27T12:16:32.344686333Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qds7wl\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"name\\\": \\\"fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:47.421017831Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.344686333Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000734e60\u003e{\n            msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-qds7wl\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"name\\\": \\\"fe40c452-f274-4427-8d6a-5d464acb5998\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:47.421017831Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.344686333Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-qds7wl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001203890\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusC...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22b50ea2e49b9f8636a928022d9c6fb867bddf916bb5e846d7ed255b2bb5febf","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"57b7fc02d64293d834d4ddb46b5e3b1b0c90895e6bf2aee448d9d619d5f34541","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"5a913a2c14e9133c66e92dca4236133e270815fdf3c4cf31f5ad59f0b56aac32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"94d395bc92cd403c8361a4d878ceae62eb9ca71d375a1c3444b0d5f3b1e35432","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e51360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-2b8h44\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\",\n      \"name\": \"b57f4712-01be-4695-929d-386468dcacfc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:46.395158145Z\",\n      \"endTime\": \"2026-02-27T12:16:32.55845307Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-2b8h44\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"name\\\": \\\"b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:46.395158145Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.55845307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e51340\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-2b8h44\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"name\\\": \\\"b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:46.395158145Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.55845307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ee9140\u003e{\n                ErrorCode: \"MultipleErr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e51360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-2b8h44\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\",\n      \"name\": \"b57f4712-01be-4695-929d-386468dcacfc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:46.395158145Z\",\n      \"endTime\": \"2026-02-27T12:16:32.55845307Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-2b8h44\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"name\\\": \\\"b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:46.395158145Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.55845307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000e51340\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-2b8h44\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"name\\\": \\\"b57f4712-01be-4695-929d-386468dcacfc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:46.395158145Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.55845307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-2b8h44/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ee9140\u003e{\n                ErrorCode: \"MultipleErr...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a169dbf582799690cc6c7941807a00ed4f37dceb52da9fd969e592d60674934b","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"69820e7f103c5c6d4cb46a97ecb688646430a77d47fc45d410566a151e2c368d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"c49450b46b3546aa94183ae38b05576ee17d860293f7a2a6fe27a4dbf1123477","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e5a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fw5p27\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\",\n      \"name\": \"98ab6997-1f34-423f-b348-c86321e9d125\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:22:17.636540715Z\",\n      \"endTime\": \"2026-02-27T12:22:47.351346292Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-fw5p27\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"name\\\": \\\"98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:22:17.636540715Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:22:47.351346292Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000c2e560\u003e{\n            msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-fw5p27\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"name\\\": \\\"98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:22:17.636540715Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:22:47.351346292Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0015d84e0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e5a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fw5p27\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\",\n      \"name\": \"98ab6997-1f34-423f-b348-c86321e9d125\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:22:17.636540715Z\",\n      \"endTime\": \"2026-02-27T12:22:47.351346292Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-fw5p27\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"name\\\": \\\"98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:22:17.636540715Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:22:47.351346292Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000c2e560\u003e{\n            msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-fw5p27\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"name\\\": \\\"98ab6997-1f34-423f-b348-c86321e9d125\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:22:17.636540715Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:22:47.351346292Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fw5p27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0015d84e0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d0dd2b6b0295b1128be54d8540853cb525e537d9e9365896c3a8d96cbc23971","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"7720ef3b84874c77615a1202d57f76d26e14065c43d49afb06950d11726187a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"0753276005f826e963dd78178216b8e991b41113f97fcb0efe291b67fb6650f7","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f20da0\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9hs4sq\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\",\n      \"name\": \"76462364-b9c8-4b34-a406-9a0d8513eb11\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:55.677677873Z\",\n      \"endTime\": \"2026-02-27T12:16:32.567400918Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9hs4sq\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"name\\\": \\\"76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:55.677677873Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.567400918Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000f20d80\u003e{\n            msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9hs4sq\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"name\\\": \\\"76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:55.677677873Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.567400918Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001683cb0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f20da0\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9hs4sq\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\",\n      \"name\": \"76462364-b9c8-4b34-a406-9a0d8513eb11\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:55.677677873Z\",\n      \"endTime\": \"2026-02-27T12:16:32.567400918Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9hs4sq\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"name\\\": \\\"76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:55.677677873Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.567400918Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000f20d80\u003e{\n            msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-9hs4sq\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"name\\\": \\\"76462364-b9c8-4b34-a406-9a0d8513eb11\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:55.677677873Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.567400918Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-9hs4sq/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001683cb0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b492cdba645190dd706f2f87d5349a72642f442e100c3e7c3c52bd3932bb3165","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"946fd461188b2cfc23372d0baa2c53801d602b11c1b98b468ccd836a5dde7172","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"6a92478828147ddb72626301aa5eda707f85c9626f8bd99ccd49e8c65bc49a8b","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009a75a0\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-kczd9q\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\",\n      \"name\": \"8bcf4081-414e-4a88-8baa-722d861e334b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:18:54.54804732Z\",\n      \"endTime\": \"2026-02-27T12:19:29.014846097Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kczd9q\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"name\\\": \\\"8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:18:54.54804732Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:19:29.014846097Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009a7580\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kczd9q\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"name\\\": \\\"8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:18:54.54804732Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:19:29.014846097Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00165d080\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009a75a0\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-kczd9q\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\",\n      \"name\": \"8bcf4081-414e-4a88-8baa-722d861e334b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:18:54.54804732Z\",\n      \"endTime\": \"2026-02-27T12:19:29.014846097Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kczd9q\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"name\\\": \\\"8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:18:54.54804732Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:19:29.014846097Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009a7580\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-kczd9q\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"name\\\": \\\"8bcf4081-414e-4a88-8baa-722d861e334b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:18:54.54804732Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:19:29.014846097Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-kczd9q/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00165d080\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a169dbf582799690cc6c7941807a00ed4f37dceb52da9fd969e592d60674934b","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"b9a7d1d18ff0f2b7ba1a945ccbe2a17d5ad8b210d0fb5cd737c206dd94deab8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"437933089286555f3098da4ddfc8aaa7d35ed1da7adfd647ccb42f03becb1128","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000804b00\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-bf2r6d\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\",\n      \"name\": \"b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:59.72983765Z\",\n      \"endTime\": \"2026-02-27T12:16:32.597146245Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-bf2r6d\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"name\\\": \\\"b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:59.72983765Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597146245Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000804ac0\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-bf2r6d\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"name\\\": \\\"b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:59.72983765Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597146245Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001148de0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000804b00\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-bf2r6d\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\",\n      \"name\": \"b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:59.72983765Z\",\n      \"endTime\": \"2026-02-27T12:16:32.597146245Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-bf2r6d\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"name\\\": \\\"b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:59.72983765Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597146245Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000804ac0\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-bf2r6d\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"name\\\": \\\"b59eb7d0-14c3-45a9-a5bc-3baa7a4d098a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:59.72983765Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.597146245Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bf2r6d/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001148de0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"838b49e8b50324bb3927086023d5f7c85bc454461244d8ef112fe4351847988d","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"be121dcd0658055946588575c36a2ae35a45afec071e4553774c1bb6e00c0c0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"297cc68bf421e66e86a97b1b84fa0dedf919e7dd45bc3cbcd3f3f17eb0e4b6a3","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009474c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8nl4rr\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\",\n      \"name\": \"22177490-e555-482e-ae86-ff318d74abb1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:21:17.276213952Z\",\n      \"endTime\": \"2026-02-27T12:21:56.634835344Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-8nl4rr\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"name\\\": \\\"22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:17.276213952Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.634835344Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009474a0\u003e{\n            msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-8nl4rr\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"name\\\": \\\"22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:17.276213952Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.634835344Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000fbb4d0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n           ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009474c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8nl4rr\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\",\n      \"name\": \"22177490-e555-482e-ae86-ff318d74abb1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:21:17.276213952Z\",\n      \"endTime\": \"2026-02-27T12:21:56.634835344Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-8nl4rr\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"name\\\": \\\"22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:17.276213952Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.634835344Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009474a0\u003e{\n            msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-8nl4rr\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"name\\\": \\\"22177490-e555-482e-ae86-ff318d74abb1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:21:17.276213952Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:21:56.634835344Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8nl4rr/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000fbb4d0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n           ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d0dd2b6b0295b1128be54d8540853cb525e537d9e9365896c3a8d96cbc23971","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"cf7c2590dacb74a9ef46851c0847c1ee1d47de42714305ade0d7251de2b4671b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"28671b564711fd286abae018fa71be2d8a2df5d16ad0824414dd8c9d89df4720","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00060c4a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cw57gb\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\",\n      \"name\": \"b486f035-e911-4179-b6b3-bb2b9455bed8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:56.141496155Z\",\n      \"endTime\": \"2026-02-27T12:16:32.486947284Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-cw57gb\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"name\\\": \\\"b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.141496155Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.486947284Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00060c480\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-cw57gb\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"name\\\": \\\"b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.141496155Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.486947284Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000de32c0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00060c4a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cw57gb\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\",\n      \"name\": \"b486f035-e911-4179-b6b3-bb2b9455bed8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:56.141496155Z\",\n      \"endTime\": \"2026-02-27T12:16:32.486947284Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-cw57gb\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"name\\\": \\\"b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.141496155Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.486947284Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00060c480\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-cw57gb\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"name\\\": \\\"b486f035-e911-4179-b6b3-bb2b9455bed8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:56.141496155Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.486947284Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-cw57gb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000de32c0\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"838b49e8b50324bb3927086023d5f7c85bc454461244d8ef112fe4351847988d","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message Region of the subnet '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"dd4c7eea0625b2dc74f35f7bce9f1327d33ffd9f681a4ff2c2dc7809629c1f92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027341290387017728","occurred_at":"2026-02-27T11:13:24Z","failed_at":"e2e","signature_id":"214662a6384053283bb06c8fdcaadeb825213530d741b62ec77cc7b730ccb6f2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000be7d80\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-vlnqhk\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\",\n      \"name\": \"a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:45.205175258Z\",\n      \"endTime\": \"2026-02-27T12:16:32.501377085Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vlnqhk\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"name\\\": \\\"a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:45.205175258Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.501377085Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000be7d60\u003e{\n            msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vlnqhk\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"name\\\": \\\"a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:45.205175258Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.501377085Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000f74210\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000be7d80\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-vlnqhk\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\",\n      \"name\": \"a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-27T12:15:45.205175258Z\",\n      \"endTime\": \"2026-02-27T12:16:32.501377085Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1' is not the same region as of the cluster\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vlnqhk\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"name\\\": \\\"a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:45.205175258Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.501377085Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000be7d60\u003e{\n            msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-vlnqhk\\\" to finish creating: GET https://rp.j7017728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus/hcpOperationStatuses/a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"name\\\": \\\"a0b24650-ec3a-4d89-8faf-c2ff49d52cbe\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-27T12:15:45.205175258Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-27T12:16:32.501377085Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Region of the subnet '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1' is not the same region as of the cluster\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"region 'westus3' of the network security group '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-vlnqhk/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000f74210\u003e{\n                ErrorCode: \"MultipleErrorsOccurred\",\n                StatusCode: 200,\n                RawResponse: {\n                ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d3bad286567b81a04e011cd13887ee4e3b89916cf4117724abf0dd166c9f2d41","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message region 'westus3' of the network security group '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name' is not the same region as of the cluster; provider Microsoft.Network","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["22010894ce78de64ead3e990516c53be92bf6fabb33ebfeb05fa683edb8b6071","22b50ea2e49b9f8636a928022d9c6fb867bddf916bb5e846d7ed255b2bb5febf","298c1e8b58b3eec0cad62cbc882f65922774d688ac5a805c14dc5324c893cbd9","6d0dd2b6b0295b1128be54d8540853cb525e537d9e9365896c3a8d96cbc23971","838b49e8b50324bb3927086023d5f7c85bc454461244d8ef112fe4351847988d","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","a169dbf582799690cc6c7941807a00ed4f37dceb52da9fd969e592d60674934b","b492cdba645190dd706f2f87d5349a72642f442e100c3e7c3c52bd3932bb3165","cad7dda9eb8f2d31e02c5973f26c8b8c1d060284fa95a731ef1903c26c11ba9a","d3bad286567b81a04e011cd13887ee4e3b89916cf4117724abf0dd166c9f2d41"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027340254221963264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3635,"pr_state":"closed","pr_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","final_merged_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T11:09:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027334101685506048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4074,"pr_state":"closed","pr_sha":"e7983a10140a6c694d743148506937010e4bfc72","final_merged_sha":"52b5dd3af1a09f5b97c6e4ae69e4544043d8ede7","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T10:44:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4225,"pr_state":"closed","pr_sha":"42b363ade2babe543637a2e48f00ff164f1c3136","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T10:44:06Z"},"failed_at":["e2e"],"failed_test_count":17,"occurrences":[{"row_id":"02fc9c40160697c7b88fb863ea0ca741a0dd599a1623157a956e4f3d9a7438a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"da336325b2a50fc611ab754711dae4992b2b14f695d9025f6a1e239d63c48d05","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011278f0\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-hnlfkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-hnlfkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011200a0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011278c0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-hnlfkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011200a0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011278f0\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-hnlfkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-hnlfkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011200a0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011278c0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-hnlfkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011200a0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-hnlfkl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"250e4a34cd1a2aea08f70d839516a5ab12224c8d66885d80810408cd00156c12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"e6a80c60ebbeda105910e2eb3d620befa92b29626eb4985657cd4dc56cbdd984","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ea29c0\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-dtj6ff\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dtj6ff\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00141c050\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ea2990\u003e{\n                msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dtj6ff\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00141c050\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ea29c0\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-dtj6ff\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dtj6ff\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00141c050\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ea2990\u003e{\n                msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dtj6ff\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00141c050\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-dtj6ff\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"2e36d4c82c99566807d0d0a15e23722a7dec3cc5528198c176c0c39b91bd1271","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"d5ac76715f9aa77f25aa51429be1af7a8174f1c1badd64edcc561103583bf00c","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011cf5c0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-sdwcw7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sdwcw7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001181a40\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011cf590\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sdwcw7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001181a40\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011cf5c0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-sdwcw7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sdwcw7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001181a40\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011cf590\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sdwcw7\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001181a40\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-sdwcw7\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"3354fe68836c217a229d5957dac35db9df85cc6b37d987b8e8ded1daaebfd77f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"09a29e92b0068f9e8f52a8ca47782861e3f31f9d1f38eeea0f02667454d6ad12","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011e0ed0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6gxxxh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6gxxxh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f02790\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011e0ea0\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6gxxxh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f02790\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011e0ed0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6gxxxh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6gxxxh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f02790\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011e0ea0\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-6gxxxh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f02790\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-6gxxxh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"37f9e7954a14971e7057e2516d51033215a597f4afd0a703f33ebc245edbe974","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"328bb5f392b5d7f71286b7a4b95fada5ef3d64f4a772f3b80c93f2859b0a2884","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00134a660\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wl48sg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00111e280\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00134a630\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wl48sg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00111e280\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00134a660\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wl48sg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00111e280\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00134a630\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-wl48sg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00111e280\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"45cc2300c53d8ae04a1c0fd34816b15e6af34206e6e824b8df95f81dac528c37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"7c911c765f58d2b5031e8ce8c9346134965a63fa293b9dcb733e456047a29546","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00111f260\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-hjd94l\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-hjd94l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dadb20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00111f230\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-hjd94l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000dadb20\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00111f260\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-hjd94l\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-hjd94l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dadb20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00111f230\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-hjd94l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000dadb20\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-hjd94l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"541cfe0f7cbfa0b116419515ab03b30ed9a50673cf9359ba652c6ab4ca04b32a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"c0d57146e05ead84ea36e1ad63621fb47fffcf11011fc7ebc6415bf48a3dee6e","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012b8900\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-strbrb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-strbrb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010bb590\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0012b88d0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-strbrb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010bb590\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012b8900\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-strbrb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-strbrb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010bb590\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0012b88d0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-strbrb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010bb590\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-strbrb\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"6fdd7cafb82469be54cb7e9cea9db7574c8e4b56294cb3f484f88c1773eb3711","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"8825dddc22955e1485036867b8081d1b3860186d7376e9f394cffa23ea2b81c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"19d1ee91c153a0300377036e3f4dcf23787e15c3f5e0f409ac66048be5cf0612","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013ff890\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xd4q9n\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-xd4q9n\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013e2350\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0013ff860\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-xd4q9n\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013e2350\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013ff890\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xd4q9n\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-xd4q9n\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013e2350\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0013ff860\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-xd4q9n\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013e2350\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-xd4q9n\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"9bace0b82d07ef88ee54c25d8f4cb8b74c57b31018b02c7105ae93806ad6b956","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"41abb96ae503f67666bf0a8cb9cb2231c36a6ca74ed23e548dbb8dd2d3dd2c07","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0006618f0\u003e: \n    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-l2z4hk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-l2z4hk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000c34440\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0006618c0\u003e{\n                msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-l2z4hk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000c34440\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0006618f0\u003e: \n    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-l2z4hk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-l2z4hk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000c34440\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0006618c0\u003e{\n                msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-l2z4hk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000c34440\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-l2z4hk\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"a3d4231d1753e08fc27900c9f58cc2ac3023819db76508a574dfac2da889dd6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"bae775d4d70694c07463236522d56c06db62ea52189d3b5ad727fce5e90e27ac","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00085cb10\u003e: \n    failed to create HCP cluster list-test-cluster-ds7sxt, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: failed waiting for cluster=\"list-test-cluster-ds7sxt\" in resourcegroup=\"cluster-listing-x9h9pr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-ds7sxt, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: failed waiting for cluster=\\\"list-test-cluster-ds7sxt\\\" in resourcegroup=\\\"cluster-listing-x9h9pr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d7a830\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00085cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-ds7sxt\\\" in resourcegroup=\\\"cluster-listing-x9h9pr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000d7a830\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00085cb10\u003e: \n    failed to create HCP cluster list-test-cluster-ds7sxt, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: failed waiting for cluster=\"list-test-cluster-ds7sxt\" in resourcegroup=\"cluster-listing-x9h9pr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-ds7sxt, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: failed waiting for cluster=\\\"list-test-cluster-ds7sxt\\\" in resourcegroup=\\\"cluster-listing-x9h9pr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d7a830\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00085cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-ds7sxt\\\" in resourcegroup=\\\"cluster-listing-x9h9pr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000d7a830\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ds7sxt in resource group cluster-listing-x9h9pr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"ae213d6a922a6896c6d75eb8b2a3078a23396b36f679679c5592cc951e659713","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"4ce195af018df0369013ee3a487681efa438741d1645df4ad13c86952d649e5d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011ff140\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-zpkckf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-zpkckf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001535210\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011ff110\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-zpkckf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001535210\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011ff140\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-zpkckf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-zpkckf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001535210\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011ff110\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-zpkckf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001535210\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-zpkckf\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"c0d205e6584c4bf589cbcde2d02e8a9f76b7c823f4197c1ab576fee9f9a5ffa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"f542993b8be0bae4401911f177d92c549724b114ec071e87ce2bfa36f1b109d6","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098eb70\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rrrj8l\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rrrj8l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011beda0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00098eb40\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rrrj8l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011beda0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098eb70\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rrrj8l\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rrrj8l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011beda0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00098eb40\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rrrj8l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011beda0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rrrj8l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"c55304c4e75666fcb35a746caaf745d888c797a925c158127d522ae8148b1835","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"e8f510d952ed376b94e1e227d2155ae7fa11372deef85e129f736aafe4b5c3bd","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146f800\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-gj9h92\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gj9h92\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0012bdf00\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00146f7d0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gj9h92\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0012bdf00\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146f800\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-gj9h92\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gj9h92\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0012bdf00\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00146f7d0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gj9h92\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0012bdf00\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-gj9h92\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"d36efbff1cd2c984bd3d7b754186fdb00a72353cfd19bd5e1cb0cfec16f23f3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"d7ed9bbad2f1140c33e6f491595bf11da8507391ccb21cf723c44e21d3658558","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f0d385a9c00, 0xc000c2b3b0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132\n                | \n                | By(\"waiting for second cluster to complete creation\")\n                \u003e _, err = poller2.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc000c2b3b0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135 @ 02/27/26 12:43:30.211\n  \"ts\"=\"2026-02-27 12:43:30.212016\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 12:43:30.212073\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6rhbc6\" \"resourceGroup\"=\"customer-rg-rdk57p\"\n  \"ts\"=\"2026-02-27 12:43:30.878205\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-6rhbc6\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 12:43:31.418389\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-krh7jl\" \"resourceGroup\"=\"customer-rg-rdk57p\"\n  \"ts\"=\"2026-02-27 12:43:31.659994\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-krh7jl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 12:43:32.242114\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 12:43:32.242259\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 12:43:32.242363\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 12:43:32.244400\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-212\"\n  \"ts\"=\"2026-02-27 12:43:32.244413\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-269\"\n  \"ts\"=\"2026-02-27 12:43:32.244426\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-122\"\n  \"ts\"=\"2026-02-27 12:43:32.244443\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-rdk57p\"\n  \"ts\"=\"2026-02-27 12:43:32.766962\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-212\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 12:43:33.116741\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-269\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 12:43:33.483086\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-122\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 12:43:33.928056\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-02-27 12:43:34.286660\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-212\" \"deployment\"=\"msi-deployment-212\"\n  \"ts\"=\"2026-02-27 12:43:34.668877\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-269\" \"deployment\"=\"msi-deployment-269\"\n  \"ts\"=\"2026-02-27 12:43:35.015527\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-122\" \"deployment\"=\"msi-deployment-122\"\n  \"ts\"=\"2026-02-27 12:43:35.439702\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-krh7jl\"\n  \"ts\"=\"2026-02-27 12:43:37.028114\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-27 12:43:37.510006\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6rhbc6\"\n  \"ts\"=\"2026-02-27 12:43:38.193769\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 12:43:38.193944\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 12:43:38.194114\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-rdk57p\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f0d385a9c00, 0xc000c2b3b0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132\n                | \n                | By(\"waiting for second cluster to complete creation\")\n                \u003e _, err = poller2.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc000c2b3b0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135 @ 02/27/26 12:43:30.211\n  \"ts\"=\"2026-02-27 12:43:30.212016\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 12:43:30.212073\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6rhbc6\" \"resourceGroup\"=\"customer-rg-rdk57p\"\n  \"ts\"=\"2026-02-27 12:43:30.878205\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-6rhbc6\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 12:43:31.418389\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-krh7jl\" \"resourceGroup\"=\"customer-rg-rdk57p\"\n  \"ts\"=\"2026-02-27 12:43:31.659994\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-krh7jl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 12:43:32.242114\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 12:43:32.242259\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 12:43:32.242363\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 12:43:32.244400\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-212\"\n  \"ts\"=\"2026-02-27 12:43:32.244413\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-269\"\n  \"ts\"=\"2026-02-27 12:43:32.244426\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-122\"\n  \"ts\"=\"2026-02-27 12:43:32.244443\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-rdk57p\"\n  \"ts\"=\"2026-02-27 12:43:32.766962\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-212\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 12:43:33.116741\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-269\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 12:43:33.483086\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-122\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 12:43:33.928056\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-02-27 12:43:34.286660\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-212\" \"deployment\"=\"msi-deployment-212\"\n  \"ts\"=\"2026-02-27 12:43:34.668877\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-269\" \"deployment\"=\"msi-deployment-269\"\n  \"ts\"=\"2026-02-27 12:43:35.015527\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-122\" \"deployment\"=\"msi-deployment-122\"\n  \"ts\"=\"2026-02-27 12:43:35.439702\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-krh7jl\"\n  \"ts\"=\"2026-02-27 12:43:37.028114\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-27 12:43:37.510006\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdk57p\" \"deployment\"=\"customer-infra-basic-hcp-cluster-6rhbc6\"\n  \"ts\"=\"2026-02-27 12:43:38.193769\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 12:43:38.193944\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 12:43:38.194114\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-rdk57p\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d9be90ca4a1cde726fc37616a7e31e3fe669d84930b9b0fea0e836c330061f2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"e4024e6b17376e6e6508b49ba7d2a81ebd4f3721aaa384f4d89c7cb32ec930c6","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fbf3b0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-sj24fh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-sj24fh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015cd890\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fbf380\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-sj24fh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015cd890\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fbf3b0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-sj24fh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-sj24fh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015cd890\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fbf380\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-sj24fh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015cd890\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-sj24fh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"eed1cf8e9faa1adb4e9c879ee89745030dae3eeb5816ef368845e63a8f283e81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027333917698166784","occurred_at":"2026-02-27T10:44:06Z","failed_at":"e2e","signature_id":"2d36ff3e7eb2e6568ce16f4dfd5aa5c24519a85064e79e926f3845e1ea0391f0","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013fad50\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-s72rsg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-s72rsg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00100d8f0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0013fad20\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-s72rsg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00100d8f0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013fad50\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-s72rsg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-s72rsg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00100d8f0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0013fad20\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-s72rsg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00100d8f0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-s72rsg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027331237286252544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4238,"pr_state":"closed","pr_sha":"72f9e678b360c792d8d10f89f71148369d0f8438","final_merged_sha":"72f9e678b360c792d8d10f89f71148369d0f8438","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T10:33:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4222/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027331106986004480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4222,"pr_state":"closed","pr_sha":"9177bd9d193220751c0db447839e5df7812e19f9","final_merged_sha":"9177bd9d193220751c0db447839e5df7812e19f9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T10:32:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027322671284359168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3635,"pr_state":"closed","pr_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","final_merged_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T09:59:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a425c3cced8c97d6d25e742979c41ee0b82783a3217c568646492f7552c6a7c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027322671284359168","occurred_at":"2026-02-27T09:59:22Z","failed_at":"provision","signature_id":"dd081b5d4a9603e30a6e136de8dd35a5e4ac47e356b2316ea6dcb9e3d0aef748","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-27T10:21:14.764Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T10:21:15.296Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:0e64ceff-a255-4ce2-81aa-ac5d1bf9c35d cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j4359168] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j4359168.hcpsvc.osadev.cloud] image:map[digest:sha256:df0aacc1e986a7eff60cd2c780998995921715b2f854b5f8f14f21f792e5b6b4 registry:arohcpsvcdev.azurecr.io repository:ci-op-dz76mcry/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-dz76mcry/pipeline:pull workloadIdentityClientId:2b32aff8-257a-42c2-96d3-df215f2b10ec workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] replicas:2 service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:1cc1da5b-ebe8-4802-9b6a-9aa28b754e16 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-02-27T10:21:15.296Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T10:21:15.615Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-02-27T10:21:16.042Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T10:21:16.043Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T10:21:16.165Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T10:26:22.325Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"resource not ready, name: sessiongate-certificate-refresher, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-27T10:26:22.325Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T10:26:22.325Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T10:26:26.557Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: resource not ready, name: sessiongate-certificate-refresher, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:0e64ceff-a255-4ce2-81aa-ac5d1bf9c35d cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j4359168] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j4359168.hcpsvc.osadev.cloud] image:map[digest:sha256:df0aacc1e986a7eff60cd2c780998995921715b2f854b5f8f14f21f792e5b6b4 registry:arohcpsvcdev.azurecr.io repository:ci-op-dz76mcry/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-dz76mcry/pipeline:pull workloadIdentityClientId:2b32aff8-257a-42c2-96d3-df215f2b10ec workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] replicas:2 service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:1cc1da5b-ebe8-4802-9b6a-9aa28b754e16 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-02-27T10:26:26.559Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6RUT0/cPhC951NYewJpvYIc+En7Ewf+VKpUqVoBp1Y9OM4jMTiedGYMAvHhq+xCSZdduqg3Z/z8PO958ooINQyhzB5ijk3t1KmrIvaouoHXLyHVc1EOqZmaVemr6/C7hDukZUV651/K+98LY4yZXFDEZGomApFAqXEK6ykpU4zgtZ3JdJdDNsLVYIsIr4HSFpJz9JEeOiR9A/gY3nqwhuvghzXjmiHtqnV3KzaIBrIhNQyRF64Tz4sc42lIdUjNAO1zjLZ6/d7QwCU8QxdMd6EGn0U38P3Zifj+3XsvwXfB48R7yrvqPotZFLzJ8+e17VxyDV6sGb/S6UZF48d6X/NuHGsP/lcflybspv4ka0scHt1AvKAY/MPS4Rjpft3jjxN0UA5erPxjI4m03S7489XV4oLyBurNnuNnhuhwIZIOY/32J7K8AlmXtd02OFmUuovn5DjHdUhhjUpmo3Mzx2Rb38/cY2bMPHXLaSp+/F/c5gqcoJBPQ5hI8WTuWzCMhg6irutNBb0HktmrnWIo703Kg/LIHpS2/O/q8GBeHs4Pj75N9s1sZrZijublgNkvnswNhWRuQ6qPQ0rgUQRSMq/JN4689awrnkzPNOy+Njo1PdUr8G4kUxOpmZoOIq4ZKIlrsKkeRuJriP8VAAD//1eFJ8WoBQAA\"\ntime=2026-02-27T10:26:26.562Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T10:26:26.563Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate-certificate-refresher, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a284dd4e312eccbb7edf0345cc86bc0bbd3bf7215354cf32ed46008df77ca55e","failure_pattern":"resource not ready, name: sessiongate-certificate-refresher, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate-certificate-refresher, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a284dd4e312eccbb7edf0345cc86bc0bbd3bf7215354cf32ed46008df77ca55e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4239/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027322304815435776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4239,"pr_state":"closed","pr_sha":"47d6ef8ee848c923c5569fcb65d6b7a7271a82be","final_merged_sha":"47d6ef8ee848c923c5569fcb65d6b7a7271a82be","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T09:57:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027319442324590592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4225,"pr_state":"closed","pr_sha":"42b363ade2babe543637a2e48f00ff164f1c3136","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T09:46:35Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"85061937436864f0b7dc0a83b44d1a66b87c4333a9d72001e248418b1060565d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027319442324590592","occurred_at":"2026-02-27T09:46:35Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T10:18:03.270Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T10:18:05.590Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T10:18:06.914Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=69f7c4622f62e632a295b0706e1b549ee46bbc7afd1957cf0bf75b6ec569d88f\ntime=2026-02-27T10:19:37.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4590592/providers/Microsoft.Resources/deployments/69f7c4622f62e632a295b0706e1b549ee46bbc7afd1957cf0bf75b6ec569d88f/operationStatuses/08584294193994438873\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4590592/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4590592/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j4590592\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report f74de8ff-463b-4c97-957c-66d3c55c1ee9:2/27/2026 10:19:16 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027313629929410560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4237,"pr_state":"closed","pr_sha":"9cf9d0988b78e3d09f927214bb2299ea73c51d4c","final_merged_sha":"9cf9d0988b78e3d09f927214bb2299ea73c51d4c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T09:23:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027313583037091840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3635,"pr_state":"closed","pr_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","final_merged_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T09:23:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8310854236680eafe50b396f14eac00b94a39b066942010436bd31e06f7e5fa2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027313583037091840","occurred_at":"2026-02-27T09:23:18Z","failed_at":"provision","signature_id":"c283ad9f7abaae1f303ce4121b34cb3fb0d4bb233af092ed1559f5ec4a947b26","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T09:47:30.635Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T09:48:02.703Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.2nPhBvLrVB/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-dz76mcry/pipeline@sha256:df0aacc1e986a7eff60cd2c780998995921715b2f854b5f8f14f21f792e5b6b4 to arohcpsvcdev.azurecr.io/ci-op-dz76mcry/pipeline:df0aacc1e986a7eff60cd2c780998995921715b2f854b5f8f14f21f792e5b6b4.\\nThe image will still be available under it's original digest sha256:df0aacc1e986a7eff60cd2c780998995921715b2f854b5f8f14f21f792e5b6b4 in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027312204021239808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4238,"pr_state":"closed","pr_sha":"72f9e678b360c792d8d10f89f71148369d0f8438","final_merged_sha":"72f9e678b360c792d8d10f89f71148369d0f8438","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T09:17:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"74735eea25e4889fef03a34324e4311570ca0e1c9177b333b2bb6d0e83c6d619","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027312204021239808","occurred_at":"2026-02-27T09:17:49Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T09:21:26.881Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T09:21:28.911Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T09:21:30.092Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=903e5da09cc4363dc3f7a8aecd3551dc9c37a85e03edb087e386fe3654bd03e7\ntime=2026-02-27T09:24:01.350Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1239808/providers/Microsoft.Resources/deployments/903e5da09cc4363dc3f7a8aecd3551dc9c37a85e03edb087e386fe3654bd03e7/operationStatuses/08584294227961177353\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1239808/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1239808/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j1239808\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 40f70052-a2dd-42a6-a66c-497fbccd4a65:2/27/2026 9:23:27 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027305318731485184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3805,"pr_state":"closed","pr_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","final_merged_sha":"278c9fc35d2143e6c8ed409ced27dd908d79caab","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T08:50:28Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"463559fe6854bfbd2cb0539a679095abc40632288b7998ca9f0c9d6cbd43955b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027305318731485184","occurred_at":"2026-02-27T08:50:28Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T09:17:15.254Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T09:17:17.545Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T09:17:18.654Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=0e5c0b3f5daaf867f38418e66ba56810e77d5509018e92c85ec9694eed14164e\ntime=2026-02-27T09:18:49.476Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1485184/providers/Microsoft.Resources/deployments/0e5c0b3f5daaf867f38418e66ba56810e77d5509018e92c85ec9694eed14164e/operationStatuses/08584294230474711808\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1485184/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1485184/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j1485184\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 35a4baf1-1851-4eae-aad8-262b78081e79:2/27/2026 9:18:28 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027305007220527104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4237,"pr_state":"closed","pr_sha":"9cf9d0988b78e3d09f927214bb2299ea73c51d4c","final_merged_sha":"9cf9d0988b78e3d09f927214bb2299ea73c51d4c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T08:49:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"35427c05bdde650c66f7deac9ae0ef90e6af0f87771100eb69926a8992de0010","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4237/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027305007220527104","occurred_at":"2026-02-27T08:49:13Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T09:12:26.243Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T09:12:28.350Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T09:12:29.623Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f50e18bfb7d4b63055fd586b14f554ab990809ca409dddaaf73dfedc7c160026\ntime=2026-02-27T09:14:30.595Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0527104/providers/Microsoft.Resources/deployments/f50e18bfb7d4b63055fd586b14f554ab990809ca409dddaaf73dfedc7c160026/operationStatuses/08584294233365600537\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0527104/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0527104/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j0527104\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 6d26be4c-6ccb-47ac-908b-074cd6f332b8:2/27/2026 9:13:52 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4222/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027303108471689216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4222,"pr_state":"closed","pr_sha":"9177bd9d193220751c0db447839e5df7812e19f9","final_merged_sha":"9177bd9d193220751c0db447839e5df7812e19f9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T08:41:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5d23c50dcbab70fe10cbb927b6d9199e0bd51476dd44079996f0069dd8cc81d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4222/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027303108471689216","occurred_at":"2026-02-27T08:41:41Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T09:05:39.438Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T09:05:41.666Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T09:05:42.953Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e78cb712d1499292674f9667ee877403769ad708eb8a4029f6e4a2cc94dd9694\ntime=2026-02-27T09:08:14.259Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1689216/providers/Microsoft.Resources/deployments/e78cb712d1499292674f9667ee877403769ad708eb8a4029f6e4a2cc94dd9694/operationStatuses/08584294237433710509\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1689216/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1689216/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j1689216\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 58abe4ce-2137-456c-baed-8f29305f49b9:2/27/2026 9:07:47 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027301188336095232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4238,"pr_state":"closed","pr_sha":"72f9e678b360c792d8d10f89f71148369d0f8438","final_merged_sha":"72f9e678b360c792d8d10f89f71148369d0f8438","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T08:34:03Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6b6f8ed38e3885e1ee724bd6989a1f803cee4e60e3fdf08d4e7bd9e265961e97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027301188336095232","occurred_at":"2026-02-27T08:34:03Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-27T09:01:03.877Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-27T09:01:06.072Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-27T09:01:07.216Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=68df04c9bca541d4adea1952abf4c3b09cefac9de7c3f2e88619c87a83de9fd6\ntime=2026-02-27T09:07:09.718Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6095232/providers/Microsoft.Resources/deployments/68df04c9bca541d4adea1952abf4c3b09cefac9de7c3f2e88619c87a83de9fd6/operationStatuses/08584294240189881759\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6095232/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6095232/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j6095232\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 75e4cdec-cfcc-4cac-b56d-31b046988a3f:2/27/2026 9:06:33 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027298978164379648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"451ccebb76e5a6fdaf3b020e7e5a47ec5992660b","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T08:25:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d0de237c6a74545ff8243169ee18922066ea70dc6941b801bd863c7ff1be57d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027298978164379648","occurred_at":"2026-02-27T08:25:16Z","failed_at":"provision","signature_id":"1e3c66f1b3aaa8e9531a0447e5fe758d4cfa54cfe1d96c0e23b03169e36088b8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"step graph","failure_text":"time=2026-02-27T09:17:25.964Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: Shell\\n  Command: ./istio.sh\\n\"\ntime=2026-02-27T09:17:28.896Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256 -o istio-1.28.3-linux-amd64.tar.gz.sha256\\n exit status 56\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"00bf42e30ff9d333f122e52fc30a4967eab6684f10cfa8840e0bee5874b06083","failure_pattern":"error running Shell Step, failed to execute shell command: + set -euo pipefail + ISTIO_NAMESPACE=aks-istio-system + echo '********** Check istio is up and running **************' ********** Check istio is up and running ************** ++ kubectl get pod","search_query":"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256 -o istio-1.28.3-linux-amd64.tar.gz.sha256\\n exit status 56"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["00bf42e30ff9d333f122e52fc30a4967eab6684f10cfa8840e0bee5874b06083"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027295681332383744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3635,"pr_state":"closed","pr_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","final_merged_sha":"667202a9b4197ac55d423e9ab819b214e5051f7f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T08:12:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"95b58acddd6c4dfe0bc89eca3ea7aeabaa9e5559bdf4c9da4ee6ea164631e50a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027295681332383744","occurred_at":"2026-02-27T08:12:10Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-02-27T08:45:29.622Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-02-27T08:45:31.189Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-02-27T08:45:32.851Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=d5769e0512b378945890bea4d234478622ce31433ee6ca57580b285b7e1cdd38\ntime=2026-02-27T09:01:37.815Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2383744-svc/providers/Microsoft.Resources/deployments/d5769e0512b378945890bea4d234478622ce31433ee6ca57580b285b7e1cdd38/operationStatuses/08584294249535324455\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2383744-svc/providers/Microsoft.Resources/deployments/profile-j6ka6jrinxygw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2383744-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j2383744-svc/resourceAssociations/nsp-j2383744-svc-ra-1ad2643f-2b6a-5f3c-af40-29a87e9667d8\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2383744-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j2383744-svc/resourceAssociations/nsp-j2383744-svc-ra-1ad2643f-2b6a-5f3c-af40-29a87e9667d8, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4152/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027290856410583040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4152,"pr_state":"closed","pr_sha":"5f79a9021da9d0841a0c96977ba86ac978d83656","final_merged_sha":"2f8a0eb8477e5204a65847041c1967c2843f8f80","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-27T07:53:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4206/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027254975003889664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4206,"pr_state":"closed","pr_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","final_merged_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T05:30:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027246034366763008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"fcf0700afe3ba0187a1a441d30f349363d59c50a","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T04:54:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"99aeb610bf2c0faf3caa3feec2ce85ba4084235f7f8268295fd84db5da998343","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027246034366763008","occurred_at":"2026-02-27T04:54:53Z","failed_at":"provision","signature_id":"7acb328ca571196613fcc24303d5ec48dcd52f7ac5349cdf5ee6de30f5b52e6c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/kustoDataConnection","test_suite":"step graph","failure_text":"time=2026-02-27T05:34:44.962Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection description=\"Step kustoDataConnection\\n  Kind: ARM\\n  Template: templates/audit-logs-data-connection.bicep\\n  Parameters: configurations/audit-logs-data-connection.tmpl.bicepparam\"\ntime=2026-02-27T05:34:45.535Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection\ntime=2026-02-27T05:34:46.363Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection deployment=eea3caabc6f0a15aa74f23fd73dcb9c21ed8c5e2d5b4860b22e8ca6410f3cf3a\ntime=2026-02-27T05:35:16.694Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/eea3caabc6f0a15aa74f23fd73dcb9c21ed8c5e2d5b4860b22e8ca6410f3cf3a/operationStatuses/08584294363995284134\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"CanNotAlterEventHubResourceId\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9341184/providers/Microsoft.EventHub/namespaces/hcp-prow-j9341184-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6763008/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs\\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6db0f157afdc9ea171276de00fcdda3a5357a5250618c220d0f9e58d2808ac27","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CanNotAlterEventHubResourceId; detail message [BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-j9341184-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-westus3-auditlogs-eh/eventhubs/aks-auditlogs; provider Microsoft.EventHub","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6db0f157afdc9ea171276de00fcdda3a5357a5250618c220d0f9e58d2808ac27"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027236156919255040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"1e25e2e5580c7906774e8dfbf8d9ba2523588cbf","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T04:15:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9715629ede6a4ff0a48387d102d46e9c40e85d689014c62f6a045b0771db9d0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027236156919255040","occurred_at":"2026-02-27T04:15:38Z","failed_at":"provision","signature_id":"8e7e33051bfd5b44466ce3db1c5bc79dfc7856a63629f54581397f53398f93fa","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-27T05:05:43.108Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-27T05:05:43.682Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:8c934114-5517-4afd-9b67-632da8015ef6 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j9255040] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j9255040.hcpsvc.osadev.cloud] image:map[digest:sha256:9ce287a8686ea9583c20df661a4d5b2a45610c265731bc7058a6809a75a04b5c registry:arohcpsvcdev.azurecr.io repository:ci-op-jjs4nklt/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-jjs4nklt/pipeline:pull workloadIdentityClientId:0a66076f-3f15-474e-841f-8c4fb8ba43cf workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] replicas:2 service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:1847a509-f680-4f2d-98b2-820afda253fc workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-02-27T05:05:43.682Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T05:05:43.944Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-02-27T05:05:44.335Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-02-27T05:05:44.335Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T05:05:44.465Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T05:10:50.960Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-27T05:10:50.960Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T05:10:50.960Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T05:10:54.751Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:8c934114-5517-4afd-9b67-632da8015ef6 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j9255040] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j9255040.hcpsvc.osadev.cloud] image:map[digest:sha256:9ce287a8686ea9583c20df661a4d5b2a45610c265731bc7058a6809a75a04b5c registry:arohcpsvcdev.azurecr.io repository:ci-op-jjs4nklt/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-jjs4nklt/pipeline:pull workloadIdentityClientId:0a66076f-3f15-474e-841f-8c4fb8ba43cf workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] replicas:2 service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:1847a509-f680-4f2d-98b2-820afda253fc workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-02-27T05:10:54.753Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5SUUWvbMBDH3/0pRJ5aiEwW2g0y+tC1g8FghK5PG3uQ5YutRtZ5d6eElHz44TppjeMk3Zt8/t9P/9Mdl3gQRcAYyQKrG5UbMWIyDxeYPYGV7y7kMxZyoRirNvTDVPAaghWElwjXxu7Dl78TpZQa/QRaOQu31mIMMhqrEQOzw1AYgYPPNucusmD1sLN0DwsXnDgMHTmnnbzUEOrS1ql5jgSpxapRvtJ8ZAF6QN+/T+/OujLBFFBBa3CfOJRhMQih90BHvJ9O0h5MDqTBg+1V1Id8cSF3oTjBygYV/8fo+TnDvIfa42b/TucbeVSvLZC4hbPNmWBBwGX7pGbJ2rE41C4UBMx71m2UEsk9m8bnHL2zmxe997jWFQg5y5rPezrJCSjl8fK/PT7OHzAOoAflu9F/n/jW0jx63+lYHb0/04+TtfRecXhI4G8EloYDQZp2HA6lplakTZTyaKmWQOaEK5cD3XlzcKNmWw/3N/nzOVnGDCiAAH9tlgknW7UugUCJq4DFVLXKQNYAQV3kRqAJX4ymk+lHPZnq6afHyfVscj27uvo1ulRpqo5pPkxm143mMtmqJ3RBLV3Ib1wIQJ0ViEG9bb7uyuvvumSrasLm75vRsaoxb8Xvg4yVx2KsKmA2RYNEyoFUtukUnwPbfwEAAP//ycxyMqgFAAA=\"\ntime=2026-02-27T05:10:54.754Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-02-27T05:10:54.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3796d4e6741d5657aa106b46b84d6ee7c621cc8a323332361ce1d435c3813f80","failure_pattern":"resource not ready, name: sessiongate, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: sessiongate, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3796d4e6741d5657aa106b46b84d6ee7c621cc8a323332361ce1d435c3813f80"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4225,"pr_state":"closed","pr_sha":"2ae66e5dac36b1aac91b8dcc77fcb24fd53d2c8f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T03:11:59Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"022247f194a88d9dcec51ce35d8eea845c89bcff7ab05dd7cf21ceb92ceffd79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"a2f550f2f288ca0fbd2086a145fe5b87283f07e9f2b259c3dce6c13eb38627fa","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009f54d0\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8j8jqc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-8j8jqc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010d6ee0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009f54a0\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-8j8jqc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010d6ee0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009f54d0\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-8j8jqc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-8j8jqc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010d6ee0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009f54a0\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-8j8jqc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010d6ee0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-8j8jqc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"0f9ffd05e424d9c5f0c431a5bfd8d3f690ee270f49ecfa9fc0bb3443bcf7408b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"b6db51df3bf295b9c55f37fe67be48107f89eb568b0612cb8af715632cd0b1a5","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ce750\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-n99j9l\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n99j9l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015f5580\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010ce720\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n99j9l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015f5580\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ce750\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-n99j9l\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n99j9l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015f5580\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010ce720\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-n99j9l\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015f5580\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n99j9l\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"10ec7197624b872609c3cd78ca0e8815358d460fa38ad32a616c71d339a097f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"44d3b20ad64783c09376c91db9aef52b3af8b7f9d4ed7b9c5862405e97c45eb8","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00112b0b0\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dj655k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-dj655k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f67bd0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00112b080\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-dj655k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f67bd0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00112b0b0\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dj655k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-dj655k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f67bd0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00112b080\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-dj655k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f67bd0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-dj655k\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"23c9f94a21ad5edbb9ca95924281bc6ada3d49eef55c7051dd42d2aab5f5e70b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"12673d4d26db7c130c9ffe598b9586548bdfa1dac04c0a1a411d696e945659cb","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001480ed0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-tb5rsr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tb5rsr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010b88a0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001480ea0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tb5rsr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010b88a0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001480ed0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-tb5rsr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tb5rsr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010b88a0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001480ea0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-tb5rsr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010b88a0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-tb5rsr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"29c0b1c1ea342ac23532634c9d6c477812e6258773c43a5737b071f6fa42152f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"8425e8446a7359967bef8cc37856eecf7de6762d7c294ea8742310d2d38d1f95","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000803290\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hf5p44\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-hf5p44\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d27510\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000803260\u003e{\n                msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-hf5p44\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000d27510\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000803290\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hf5p44\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-hf5p44\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d27510\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000803260\u003e{\n                msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-hf5p44\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000d27510\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-hf5p44\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"3eb359b3e063fd478ae934d0fd4b64ae09bf8ccf34dc66ad3f824e5a7b78e6b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"7dfd98154024bd1d7c5b54edc1e4a59af45e89df95c801e2a83d86ef69dc38e2","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"0.1({0x7f9b2039f260, 0xc000904240})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:80\n                | \n                | By(\"creating HCP cluster: \" + clusterName)\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc000904240?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 99\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86 @ 02/27/26 05:36:55.442\n  \"ts\"=\"2026-02-27 05:36:55.443068\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 05:36:55.443163\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-m5gvlr-mwcsts\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:36:56.227033\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-m5gvlr-mwcsts\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:56.766314\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-8dv8xs-dqm829\" \"resourceGroup\"=\"cluster-listing-dzhbvx\"\n  \"ts\"=\"2026-02-27 05:36:56.922073\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-8dv8xs-dqm829\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:57.515098\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.515198\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.515259\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 05:36:57.516325\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-67\"\n  \"ts\"=\"2026-02-27 05:36:57.516352\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:36:57.516379\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-275\"\n  \"ts\"=\"2026-02-27 05:36:57.516337\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-dzhbvx\"\n  \"ts\"=\"2026-02-27 05:36:58.008719\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-67\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:58.433640\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-8cn59b\" \"deployment\"=\"pooledNonMsiScopedAssignments-fux5nzle55hgk\"\n  \"ts\"=\"2026-02-27 05:36:58.814821\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-275\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:59.217189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-dzhbvx\" \"deployment\"=\"pooledNonMsiScopedAssignments-xjplj2obx3rl4\"\n  \"ts\"=\"2026-02-27 05:36:59.614165\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-67\" \"deployment\"=\"msi-deployment-67\"\n  \"ts\"=\"2026-02-27 05:37:00.002164\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-8cn59b\" \"deployment\"=\"customer-infra-list-test-cluster-m5gvlr-mwcsts\"\n  \"ts\"=\"2026-02-27 05:37:00.394239\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-275\" \"deployment\"=\"msi-deployment-275\"\n  \"ts\"=\"2026-02-27 05:37:00.830897\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-dzhbvx\" \"deployment\"=\"customer-infra-list-test-cluster-8dv8xs-dqm829\"\n  \"ts\"=\"2026-02-27 05:37:02.523218\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 05:37:02.523369\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 05:37:02.523503\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-dzhbvx\"\n  \"ts\"=\"2026-02-27 05:37:02.523587\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:46:20.542587\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:46:20.542624\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n0.1({0x7f9b2039f260, 0xc000904240})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:80\n                | \n                | By(\"creating HCP cluster: \" + clusterName)\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc000904240?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 99\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86 @ 02/27/26 05:36:55.442\n  \"ts\"=\"2026-02-27 05:36:55.443068\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 05:36:55.443163\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-m5gvlr-mwcsts\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:36:56.227033\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-m5gvlr-mwcsts\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:56.766314\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-list-test-cluster-8dv8xs-dqm829\" \"resourceGroup\"=\"cluster-listing-dzhbvx\"\n  \"ts\"=\"2026-02-27 05:36:56.922073\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-list-test-cluster-8dv8xs-dqm829\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:57.515098\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.515198\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b860a86958f57179c37d87e5f83be5384f83e5bf95c88d961f55ca4e4ca1bbbb.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.515259\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 05:36:57.516325\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-67\"\n  \"ts\"=\"2026-02-27 05:36:57.516352\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:36:57.516379\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-275\"\n  \"ts\"=\"2026-02-27 05:36:57.516337\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"cluster-listing-dzhbvx\"\n  \"ts\"=\"2026-02-27 05:36:58.008719\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-67\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:58.433640\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-8cn59b\" \"deployment\"=\"pooledNonMsiScopedAssignments-fux5nzle55hgk\"\n  \"ts\"=\"2026-02-27 05:36:58.814821\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-275\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:59.217189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-dzhbvx\" \"deployment\"=\"pooledNonMsiScopedAssignments-xjplj2obx3rl4\"\n  \"ts\"=\"2026-02-27 05:36:59.614165\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-67\" \"deployment\"=\"msi-deployment-67\"\n  \"ts\"=\"2026-02-27 05:37:00.002164\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-8cn59b\" \"deployment\"=\"customer-infra-list-test-cluster-m5gvlr-mwcsts\"\n  \"ts\"=\"2026-02-27 05:37:00.394239\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-275\" \"deployment\"=\"msi-deployment-275\"\n  \"ts\"=\"2026-02-27 05:37:00.830897\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"cluster-listing-dzhbvx\" \"deployment\"=\"customer-infra-list-test-cluster-8dv8xs-dqm829\"\n  \"ts\"=\"2026-02-27 05:37:02.523218\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 05:37:02.523369\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 05:37:02.523503\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-dzhbvx\"\n  \"ts\"=\"2026-02-27 05:37:02.523587\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:46:20.542587\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\n  \"ts\"=\"2026-02-27 05:46:20.542624\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"cluster-listing-8cn59b\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"4637345c1571a7b2cc41188c54e783f1f68d48f52fd077623ded49d15c0dacc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"26600eef9f2f265429eeb1c77a2197c92f49c2d9c516acfb0f9b4d01136a3405","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"r.go:284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7fc3ea458d18, 0xc00090f4d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc00090f4d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 15\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 02/27/26 05:36:55.421\n  \"ts\"=\"2026-02-27 05:36:55.421654\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 05:36:55.421735\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-jbmd7v\" \"resourceGroup\"=\"customer-rg-6glht6\"\n  \"ts\"=\"2026-02-27 05:36:56.073863\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-jbmd7v\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:56.715773\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-kvzkbc\" \"resourceGroup\"=\"customer-rg-6glht6\"\n  \"ts\"=\"2026-02-27 05:36:56.888381\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-kvzkbc\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:57.558300\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.558421\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.558497\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 05:36:57.560269\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-2\"\n  \"ts\"=\"2026-02-27 05:36:57.560499\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-6glht6\"\n  \"ts\"=\"2026-02-27 05:36:57.560536\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-277\"\n  \"ts\"=\"2026-02-27 05:36:57.560520\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-164\"\n  \"ts\"=\"2026-02-27 05:36:58.190245\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-2\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:58.610937\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-02-27 05:36:58.933018\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-277\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:59.288101\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-164\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:59.662456\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-2\" \"deployment\"=\"msi-deployment-2\"\n  \"ts\"=\"2026-02-27 05:37:00.087312\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-kvzkbc\"\n  \"ts\"=\"2026-02-27 05:37:00.452564\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-277\" \"deployment\"=\"msi-deployment-277\"\n  \"ts\"=\"2026-02-27 05:37:00.910441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-164\" \"deployment\"=\"msi-deployment-164\"\n  \"ts\"=\"2026-02-27 05:37:01.661105\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-27 05:37:02.811375\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"customer-infra-basic-hcp-cluster-jbmd7v\"\n  \"ts\"=\"2026-02-27 05:37:03.369104\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 05:37:03.369255\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 05:37:03.369366\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-6glht6\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nr.go:284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7fc3ea458d18, 0xc00090f4d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc00090f4d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 15\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 02/27/26 05:36:55.421\n  \"ts\"=\"2026-02-27 05:36:55.421654\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 05:36:55.421735\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-jbmd7v\" \"resourceGroup\"=\"customer-rg-6glht6\"\n  \"ts\"=\"2026-02-27 05:36:56.073863\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-jbmd7v\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:56.715773\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-kvzkbc\" \"resourceGroup\"=\"customer-rg-6glht6\"\n  \"ts\"=\"2026-02-27 05:36:56.888381\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-kvzkbc\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 05:36:57.558300\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.558421\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-27 05:36:57.558497\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 05:36:57.560269\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-2\"\n  \"ts\"=\"2026-02-27 05:36:57.560499\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-6glht6\"\n  \"ts\"=\"2026-02-27 05:36:57.560536\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-277\"\n  \"ts\"=\"2026-02-27 05:36:57.560520\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-164\"\n  \"ts\"=\"2026-02-27 05:36:58.190245\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-2\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:58.610937\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-02-27 05:36:58.933018\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-277\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:59.288101\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-164\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 05:36:59.662456\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-2\" \"deployment\"=\"msi-deployment-2\"\n  \"ts\"=\"2026-02-27 05:37:00.087312\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-kvzkbc\"\n  \"ts\"=\"2026-02-27 05:37:00.452564\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-277\" \"deployment\"=\"msi-deployment-277\"\n  \"ts\"=\"2026-02-27 05:37:00.910441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-164\" \"deployment\"=\"msi-deployment-164\"\n  \"ts\"=\"2026-02-27 05:37:01.661105\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-27 05:37:02.811375\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-6glht6\" \"deployment\"=\"customer-infra-basic-hcp-cluster-jbmd7v\"\n  \"ts\"=\"2026-02-27 05:37:03.369104\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 05:37:03.369255\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 05:37:03.369366\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-6glht6\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"65e0bcd4bd0a9c000dfbf6055bfb8ce1b90c20ea296d1f0a64521a56c4235fed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"c08ae5de6f714b301cfd08c10ced45b4f3d524cb8384a7934293688858efc39e","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e5ac60\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-pf5wk5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pf5wk5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000097ae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e5ac30\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pf5wk5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000097ae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e5ac60\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-pf5wk5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pf5wk5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000097ae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e5ac30\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pf5wk5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000097ae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pf5wk5\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"7eb2d044006d2dd3491a6e0b4fc828a32dc3f50c9cbaac05f34d70ac245782e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"a8e18c2c605b7bc00c0f07e8179e91b4e868d354ecfba9c0750606ff4333f1e3","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00113b080\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5nrmv6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-5nrmv6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ec3040\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00113b050\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-5nrmv6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ec3040\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00113b080\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5nrmv6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-5nrmv6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ec3040\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00113b050\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-5nrmv6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ec3040\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-5nrmv6\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"a0a86b823218865f288c9d9ba9ab4712a2498fa84ab9f12a64dfd47cc2cc541f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"b170fff3f449da03b50918ac7877ff4a145eeb577f6fc021beb639522112f6e2","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014ce720\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-xwtlrk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-xwtlrk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fc3370\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014ce6f0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-xwtlrk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fc3370\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014ce720\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-xwtlrk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-xwtlrk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fc3370\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014ce6f0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-xwtlrk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fc3370\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-xwtlrk\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"a46d20df657efdb552fa61c139c34405a4baae315a74de7845268807d78fc9fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"4052c939ecfe9f1966131e97efbe7192ed7b24be520197ee178bf9345cbceefd","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00114b500\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7727wj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-7727wj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fe8310\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00114b4d0\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-7727wj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fe8310\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00114b500\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7727wj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-7727wj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fe8310\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00114b4d0\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-7727wj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fe8310\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-7727wj\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"a90cd0d3781a6d63662486c7f5d85c463f80909a95ca3588efedc2ed97f2f790","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"00be31569222bfab116104a8ab8d4fe5425bd54cba15965f93c8601366373d89","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001165410\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d64w46\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-d64w46\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00120bae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011653e0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-d64w46\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00120bae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001165410\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d64w46\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-d64w46\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00120bae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011653e0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-d64w46\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00120bae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-d64w46\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"aa030727aa2a139fce569a41535c85aaeb82c747203d0e50309c44a6e0cdd477","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"25e7d80babb9df76a31da2ff28fd120298b6fa8e0c8ceb8ff02440035de9a120","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012832f0\u003e: \n    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-7ggbvj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7ggbvj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010351d0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0012832c0\u003e{\n                msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7ggbvj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010351d0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012832f0\u003e: \n    failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-7ggbvj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7ggbvj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010351d0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0012832c0\u003e{\n                msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-7ggbvj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010351d0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-7ggbvj\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"b1248fcc6f5c85e00d00ef2eb912a648524a20f6b6a4b71dbb6370a954839d69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"3799c688f32d94ed23ab77ecafeefcd0de53489c3a9a7ad03cf357e80206ec66","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015fb590\u003e: \n    failed to create HCP cluster ea-listtncz4s, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: failed waiting for cluster=\"ea-listtncz4s\" in resourcegroup=\"ea-list-gbf6qr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-listtncz4s, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: failed waiting for cluster=\\\"ea-listtncz4s\\\" in resourcegroup=\\\"ea-list-gbf6qr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015e4c10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0015fb560\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-listtncz4s\\\" in resourcegroup=\\\"ea-list-gbf6qr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015e4c10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015fb590\u003e: \n    failed to create HCP cluster ea-listtncz4s, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: failed waiting for cluster=\"ea-listtncz4s\" in resourcegroup=\"ea-list-gbf6qr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-listtncz4s, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: failed waiting for cluster=\\\"ea-listtncz4s\\\" in resourcegroup=\\\"ea-list-gbf6qr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015e4c10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0015fb560\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-listtncz4s\\\" in resourcegroup=\\\"ea-list-gbf6qr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015e4c10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtncz4s in resource group ea-list-gbf6qr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}},{"row_id":"c68336daa4ad67bc7f57addc70834fc6d4a87d997e3008cd3fbd9c650eaebbdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"7dc26f820fd2928c111033bd0ce08af1bae6042a6cbe5fb5bf76d461b5be1d01","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"cf417e0508e31ab3e1fd5ec0bbe8b26ab573f8405c2e26f27271e0098485e8aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027220140248010752","occurred_at":"2026-02-27T03:11:59Z","failed_at":"e2e","signature_id":"16bae46d3d264cf6f0648acebe66d34a47c31eb2cfb9032639deb784ecf42d33","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014b5140\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4g74g6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4g74g6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001658490\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014b5110\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4g74g6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001658490\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014b5140\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4g74g6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4g74g6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001658490\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014b5110\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-4g74g6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001658490\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-4g74g6\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wl48sg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-wl48sg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4206/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027218026868248576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4206,"pr_state":"closed","pr_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","final_merged_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T03:03:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027203468065771520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4232,"pr_state":"closed","pr_sha":"4dd0397266a29ae004677e31f2f9e424aa671200","final_merged_sha":"4dd0397266a29ae004677e31f2f9e424aa671200","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-27T02:05:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a5969bd820a09f763d4a2ba29b766d7f73a94d8775bff4bd0769025605e1a2a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027203468065771520","occurred_at":"2026-02-27T02:05:45Z","failed_at":"provision","signature_id":"f25a8f6afd29af1c8da816d106f04e2b1fec3283ad18587867c27bff4c571194","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T02:49:33.416Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T02:50:11.826Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.nXFdH6BEsg/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-w6w2i9zr/pipeline@sha256:15fdb8bcfeb3eeb4d0246898b617f3abfa4eafb8966ac002f3e53f7d5d027ba2 to arohcpsvcdev.azurecr.io/ci-op-w6w2i9zr/pipeline:15fdb8bcfeb3eeb4d0246898b617f3abfa4eafb8966ac002f3e53f7d5d027ba2.\\nThe image will still be available under it's original digest sha256:15fdb8bcfeb3eeb4d0246898b617f3abfa4eafb8966ac002f3e53f7d5d027ba2 in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027202767583449088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4232,"pr_state":"closed","pr_sha":"4dd0397266a29ae004677e31f2f9e424aa671200","final_merged_sha":"4dd0397266a29ae004677e31f2f9e424aa671200","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T02:02:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027197237192298496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4225,"pr_state":"closed","pr_sha":"2ae66e5dac36b1aac91b8dcc77fcb24fd53d2c8f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T01:40:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"52e7b21e6499c7ac492bc55851c1eaf184cc368ce1f8c04860d5ff77b2560d53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027197237192298496","occurred_at":"2026-02-27T01:40:59Z","failed_at":"provision","signature_id":"846d2c52bbab25bf9dbee6a2186a62cf48f071ac5f91ca56908b6b08a5d49380","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-27T02:26:47.671Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-27T02:27:18.176Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.HVDbOI9WEt/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-zzsmsybr/pipeline@sha256:d36350276c185fa745197826e9f61a293a89755db8fe6899056d7366668535a0 to arohcpsvcdev.azurecr.io/ci-op-zzsmsybr/pipeline:d36350276c185fa745197826e9f61a293a89755db8fe6899056d7366668535a0.\\nThe image will still be available under it's original digest sha256:d36350276c185fa745197826e9f61a293a89755db8fe6899056d7366668535a0 in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027176692673417216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"b34811a6627cd8d7208234087e80ddf2a4b65915","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-27T00:19:21Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"77529e3ecd0dde20691e9d629ab074e8a8d30bb62c962c335d05c2448404aa9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027176692673417216","occurred_at":"2026-02-27T00:19:21Z","failed_at":"provision","signature_id":"7acb328ca571196613fcc24303d5ec48dcd52f7ac5349cdf5ee6de30f5b52e6c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/kustoDataConnection","test_suite":"step graph","failure_text":"time=2026-02-27T01:10:55.153Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection description=\"Step kustoDataConnection\\n  Kind: ARM\\n  Template: templates/audit-logs-data-connection.bicep\\n  Parameters: configurations/audit-logs-data-connection.tmpl.bicepparam\"\ntime=2026-02-27T01:10:55.580Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection\ntime=2026-02-27T01:10:56.264Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection deployment=337f897e348f0838d149dd8f29eb4adc1035a9e343574bd88df3476966dc791e\ntime=2026-02-27T01:11:26.354Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=kustoDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/337f897e348f0838d149dd8f29eb4adc1035a9e343574bd88df3476966dc791e/operationStatuses/08584294522295353112\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"CanNotAlterEventHubResourceId\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9341184/providers/Microsoft.EventHub/namespaces/hcp-prow-j9341184-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3417216/providers/Microsoft.EventHub/namespaces/hcp-prow-j3417216-auditlogs-eh/eventhubs/aks-auditlogs\\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"400e06dcfd958b3d8511c0fde456840034651e306ff07a3f5f0d6e971b26ccd2","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CanNotAlterEventHubResourceId; detail message [BadRequest] Cannot alter event hub resource id. Existing: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-j9341184-auditlogs-eh/eventhubs/aks-auditlogs, Given: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.EventHub/namespaces/hcp-prow-j3417216-auditlogs-eh/eventhubs/aks-auditlogs; provider Microsoft.EventHub","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["400e06dcfd958b3d8511c0fde456840034651e306ff07a3f5f0d6e971b26ccd2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2027174133657243648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4212,"pr_state":"closed","pr_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","final_merged_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-27T00:09:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}