{"meta":{"date":"2026-03-10","timezone":"UTC","generated_at":"2026-11-01T00:08:20Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":86,"failed_runs":40,"runs_with_occurrences":40,"runs_with_failure_pattern_matches":29,"runs_with_unmatched_occurrences":11,"runs_without_occurrence_rows":46,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4376/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031517502743252992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4376,"pr_state":"closed","pr_sha":"0e576feee6204e903fe01fe78e4acd401f56f38f","final_merged_sha":"0b33790f453ea37c143074eb4e7d7a235407d207","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T23:48:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031511837974990848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4381,"pr_state":"closed","pr_sha":"240cba2542c3ec7bf793428a2afaac082748b26e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T23:25:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031500306147184640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4371,"pr_state":"closed","pr_sha":"11ecb878ec9a3501bb7ba58e755d71f6abf8de21","final_merged_sha":"e0b528c4354bf750dd12c0c682e19c640905138e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T22:39:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"77588cf1e17a4a28f0857e623a7deffeb7ba089688bd3c5cae8b037fc0a4761b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031500306147184640","occurred_at":"2026-03-10T22:39:51Z","failed_at":"provision","signature_id":"1afb28c8e157da381239a18ffe2b4a0af4e45e4c32936e2ba90253d7fe3da81c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RouteMonitorOperator/global/mirror-blackbox-image","test_suite":"step graph","failure_text":"time=2026-03-10T23:33:28.873Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-blackbox-image description=\"Step mirror-blackbox-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e routeMonitorOperator.blackboxExporterImage.registry \u003cnil\u003e}:{\u003cnil\u003e routeMonitorOperator.blackboxExporterImage.repository \u003cnil\u003e}@{\u003cnil\u003e routeMonitorOperator.blackboxExporterImage.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T23:33:53.217Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-blackbox-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n sock = connection.create_connection(\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n response = self._make_request(\\n ^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n raise new_e\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n self._validate_conn(conn)\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n conn.connect()\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n self.sock = sock = self._new_conn()\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n resp = conn.urlopen(\\n ^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n retries = retries.increment(\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n cmd_result = self.invocation.execute(args)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n raise ex\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n results.append(self._run_job(expanded_arg, cmd_copy))\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n result = cmd_copy(params)\\n ^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n return self.handler(*args, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n return op(**command_args)\\n ^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n login_server, _, password = get_login_credentials(\\n ^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n return _get_credentials(cmd,\\n ^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n ^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 276, in _get_aad_token\\n token_params = _handle_challenge_phase(\\n ^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 100, in _handle_challenge_phase\\n challenge = requests.get(request_url, verify=not should_disable_connection_verify())\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 73, in get\\n return request(\\\"get\\\", url, params=params, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n return session.request(method=method, url=url, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n resp = self.send(prep, **send_kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n r = adapter.send(request, **kwargs)\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031492131998142464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T22:03:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491663913816064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T21:58:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9e51814bd3feddb4cf5c81b8be61f21f75a5bcd9752f4e897ee8ceee24d7099f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491663913816064","occurred_at":"2026-03-10T21:58:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491511228567552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T21:55:42Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"60e949307086b96a7e196cfd052ebac2e49f0af31dfa93ee0185cdc2be19bbc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491511228567552","occurred_at":"2026-03-10T21:55:42Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491462608195584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T21:52:59Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"621106cf238cf694d8c89b8fff1c7d805bfeb869323bf8865223074efddd2df7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491462608195584","occurred_at":"2026-03-10T21:52:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491374502645760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"open","pr_sha":"2dcd46561fc1343d22436901c4bb23de90beb172","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T21:52:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031491288083206144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T21:50:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031478832665464832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4396,"pr_state":"closed","pr_sha":"66a8da73f74ada368832de4386867641900d90f9","final_merged_sha":"732c1753890fbef90e84268733f7e4d18f32e691","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T21:14:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e60f8f562efdc6cab9486af65d986b77962dc366f1328ec11c927419ee1770a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031478832665464832","occurred_at":"2026-03-10T21:14:29Z","failed_at":"provision","signature_id":"5e8b4532bf95340a6deceb760fe6ac0df2e2775dfbc36e82dc16d755a8fb0d51","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-10T21:47:40.879Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-10T21:47:42.919Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-927792105.bicepparam level=Error range=L20:32-L20:33 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-10T21:47:42.919Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-927792105.bicepparam level=Error range=L20:32-L20:33 code=BCP019\ntime=2026-03-10T21:47:42.919Z level=ERROR msg=\"invalid .bicepparam\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-927792105.bicepparam level=Error range=L21:31-L21:32 code=BCP019 err=\"Expected a new line character at this location.\"\ntime=2026-03-10T21:47:42.919Z level=INFO msg=\"Expected a new line character at this location.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra source=/opt/app-root/src/github.com/Azure/ARO-HCP/dev-infrastructure/configurations/bicep-params-927792105.bicepparam level=Error range=L21:31-L21:32 code=BCP019\ntime=2026-03-10T21:47:42.920Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e","failure_pattern":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors","search_query":"failed to run ARM step: failed to transform Bicep to ARM: failed to build params: failed to compile bicepparam: 2 errors"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32e3454e6f92d487393db8e0bbde176360d6dafa4345f283cdf12af6bf55571e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031475506083270656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T21:00:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031472430676971520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4383,"pr_state":"closed","pr_sha":"9128c3e9a3727d0be6602fb2b928876fd434f14c","final_merged_sha":"9128c3e9a3727d0be6602fb2b928876fd434f14c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T20:49:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4399/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031472210912219136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4399,"pr_state":"closed","pr_sha":"c93b50a5a0ae5183dd6c572e3734449d60bd0d00","final_merged_sha":"8a1ae2ac7a78b624556d264008ca37e8c812709f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T20:48:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4376/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031471249284141056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4376,"pr_state":"closed","pr_sha":"2aab112fa30514e7f917259f93ae8443be32c0df","final_merged_sha":"0b33790f453ea37c143074eb4e7d7a235407d207","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T20:44:23Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"202867706949070fce03a69015f6d53aff80e2b66b5978f68b87bb716d11d54e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4376/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031471249284141056","occurred_at":"2026-03-10T20:44:23Z","failed_at":"e2e","signature_id":"7770e8d6d334fd9b906a08d2fe59aa9c9244976e2a64c7459dc4cfb42780f423","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00067ed00\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-c4qvzv\" to finish creating: GET https://rp.j4141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\",\n \"name\": \"350a4761-0c47-48d6-8823-946d38f9fe6b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T21:39:55.404928229Z\",\n \"endTime\": \"2026-03-10T21:40:53.386830866Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-c4qvzv\\\" to finish creating: GET https://rp.j4141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"name\\\": \\\"350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T21:39:55.404928229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T21:40:53.386830866Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00067ece0\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-c4qvzv\\\" to finish creating: GET https://rp.j4141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"name\\\": \\\"350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T21:39:55.404928229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T21:40:53.386830866Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00067ed00\u003e: \n failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-c4qvzv\" to finish creating: GET https://rp.j4141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\",\n \"name\": \"350a4761-0c47-48d6-8823-946d38f9fe6b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T21:39:55.404928229Z\",\n \"endTime\": \"2026-03-10T21:40:53.386830866Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-c4qvzv\\\" to finish creating: GET https://rp.j4141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"name\\\": \\\"350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T21:39:55.404928229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T21:40:53.386830866Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00067ece0\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-c4qvzv\\\" to finish creating: GET https://rp.j4141056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"name\\\": \\\"350a4761-0c47-48d6-8823-946d38f9fe6b\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T21:39:55.404928229Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T21:40:53.386830866Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-60/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4398/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031468828357038080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4398,"pr_state":"open","pr_sha":"06408b44aade23d9c2a20924d1d27f58cbb8ce19","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T20:34:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466924021387264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4397,"pr_state":"closed","pr_sha":"f7dc8e180d3210270c0c21e0965aa7ba711ab018","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T20:27:06Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3de44a607ac1f201a85d649642499b731dfdc82f72e415102465aa4b647484eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466924021387264","occurred_at":"2026-03-10T20:27:06Z","failed_at":"e2e","signature_id":"3a5c9225f9647e371d5effdf41e1d105f05cd6d3d51e037a978aa1f1ecffb8e6","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000df0dc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tq24zj\" to finish creating: GET https://rp.j1387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\",\n \"name\": \"b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T21:43:34.668466002Z\",\n \"endTime\": \"2026-03-10T21:44:33.526415167Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-tq24zj\\\" to finish creating: GET https://rp.j1387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\\\",\\n \\\"name\\\": \\\"b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T21:43:34.668466002Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T21:44:33.526415167Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0004d7e00\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"2206b29d-f111-49c3-bab4-1a07c00f8fc4\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 21:44:35 GMT\",\n ],\n \"Content-Length\": [\"1563\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000df0cc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 98, 56, 49, 48, 51, 98, 99, 100, 45, 52, 100, 99, 50, 45, 52, 99, 51, 102, 45, 56, 97, 52, 52, 45, 98, 51, 102, 102, 54, 101, 56, 48, 54, 100, 52, 49, 34, 44, 10, 32, 32, 32, 32, 34, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000df0dc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tq24zj\" to finish creating: GET https://rp.j1387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\",\n \"name\": \"b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T21:43:34.668466002Z\",\n \"endTime\": \"2026-03-10T21:44:33.526415167Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-tq24zj\\\" to finish creating: GET https://rp.j1387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\\\",\\n \\\"name\\\": \\\"b8103bcd-4dc2-4c3f-8a44-b3ff6e806d41\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T21:43:34.668466002Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T21:44:33.526415167Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-193/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0004d7e00\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"2206b29d-f111-49c3-bab4-1a07c00f8fc4\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 21:44:35 GMT\",\n ],\n \"Content-Length\": [\"1563\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000df0cc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 98, 56, 49, 48, 51, 98, 99, 100, 45, 52, 100, 99, 50, 45, 52, 99, 51, 102, 45, 56, 97, 52, 52, 45, 98, 51, 102, 102, 54, 101, 56, 48, 54, 100, 52, 49, 34, 44, 10, 32, 32, 32, 32, 34, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"730aedc1e2f03e62758aaba9b43c4ccdd249c551","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T20:26:43Z"},"failed_at":["e2e"],"failed_test_count":21,"occurrences":[{"row_id":"11e1a19208c5e8c94a2d30cdcb8e5d5a30c6a0b9815f3ccbc0b8fbf4f76fb313","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"9e6fd2b26d9f17727b0e09d17eb4c877ac6753be4ab0b2928934f6fe5064d6f4","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d248b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p99sfc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001570c80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-p99sfc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001570c60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-p99sfc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001570c40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-p99sfc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d248b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p99sfc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001570c80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-p99sfc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001570c60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-p99sfc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001570c40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-p99sfc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"17f3a57ab08cb3bbfccc1f705ddd0b39b19fb52f382f66ea01b35ff02cd4b0d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"1a9ee6a843e633dd13141660af2f85b0db721191c477d62c9958ff8f376d0517","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"617e7f492c74519615030c59a11a41d0d9087aa75c1391c64ef017c5d71975c7","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ea6258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2m4dst\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00150ad80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m4dst\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00150ad60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m4dst\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00150ad40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m4dst\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ea6258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2m4dst\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00150ad80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m4dst\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00150ad60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m4dst\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00150ad40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2m4dst\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"1cd8837f4fc1152ee93b6c5cac0e0a99b38058e8d373e51f0a913c008f5141d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"93f0d9af3dd6ebffc2d414acffe97da608719986fc1dfca51b0f557ef1f2eaf9","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:102]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e1c810\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-gk98xl, error: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-gk98xl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-gk98xl, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-gk98xl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012e4b70\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-gk98xl\",\n },\n \u003c*fmt.wrapError | 0xc000aae4c0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-gk98xl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:102]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e1c810\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-gk98xl, error: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-gk98xl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-gk98xl, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-gk98xl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012e4b70\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group rg-negative-tests-gk98xl\",\n },\n \u003c*fmt.wrapError | 0xc000aae4c0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-gk98xl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"3d9f64ea24742ec5ccac381f7aa5bf9a131596abb9a6f2aa94dcad3760684b0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"e493e31a1203a6b42461bc0bc40c97061297f858addc9e2327f95513a065b31e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ee6138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-fqgjfn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a19c20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqgjfn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a19c00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqgjfn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a19be0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqgjfn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ee6138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-fqgjfn\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a19c20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqgjfn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a19c00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqgjfn\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000a19be0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqgjfn\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"3fb31fd34598ff79e537d2ff928635b6812aa9e0f04fe44b729373dfb357db00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"ca74fd714cf260a56dd410a998c20e38fcc075634c2cf90cdf4893e9164743c0","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ea9398\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-tdzsft\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001038ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-tdzsft\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001038c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-tdzsft\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001038c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-tdzsft\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ea9398\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-tdzsft\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001038ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-tdzsft\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001038c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-tdzsft\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001038c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-tdzsft\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"579ea1019556e84fab8354c864b716ade0fcfbfded59a44a6ad67e2019311e71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"c06dffac0f5d24f4ed9cec9c1d55e999efb8c755439be2fd3b8ad2e74efe1067","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001334240\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-22fc9c\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012fbd50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c\",\n },\n \u003c*fmt.wrapError | 0xc00136a140\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-22fc9c\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001334240\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-22fc9c\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012fbd50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c\",\n },\n \u003c*fmt.wrapError | 0xc00136a140\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-22fc9c\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"8d903274fec56c6bee95a96faa9a4344957cd7b8893c08f8f20a48a659bb523f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"03da1552d3a0f69b6bf9abf01b45b1f82f0a4a2d7eebb2e821984f3a7896e116","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00134ee70\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lqwfph, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-lqwfph\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lqwfph, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lqwfph\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001509300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lqwfph\",\n },\n \u003c*fmt.wrapError | 0xc000e96d80\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lqwfph\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00134ee70\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lqwfph, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-lqwfph\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lqwfph, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lqwfph\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001509300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lqwfph\",\n },\n \u003c*fmt.wrapError | 0xc000e96d80\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lqwfph\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"99ad84bfbb38e14e9e6bd1a0e57103a7774d12e07859d24af883372e0ac50a28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"d498043283c937aaa335d3a959f333b1eb1e9068c6bab1fe52c6ee77bccce5fd","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001406168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-4sswbg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001548da0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4sswbg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001548d80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4sswbg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001548d60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4sswbg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001406168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-4sswbg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001548da0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4sswbg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001548d80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4sswbg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001548d60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-4sswbg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"9f174077a7585af3d386b32b93cf02c8ce8bee336a1e9366c003a6e836ba5264","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"3be0e3234eeceafdf5246cc1639f079588ab35e75cd8a2ae77b817abd56c7d00","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00020b170\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listqn5zgz\" in resourcegroup=\"ea-list-m4v8xd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f891a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listqn5zgz\\\" in resourcegroup=\\\"ea-list-m4v8xd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f89180\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listqn5zgz\\\" in resourcegroup=\\\"ea-list-m4v8xd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f89160\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listqn5zgz\\\" in resourcegroup=\\\"ea-list-m4v8xd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00020b170\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listqn5zgz\" in resourcegroup=\"ea-list-m4v8xd\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f891a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listqn5zgz\\\" in resourcegroup=\\\"ea-list-m4v8xd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f89180\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listqn5zgz\\\" in resourcegroup=\\\"ea-list-m4v8xd\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f89160\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listqn5zgz\\\" in resourcegroup=\\\"ea-list-m4v8xd\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"b125aa799e5d5e4c1578c01dbee7b2abb1b1be2262f0748c18b3136a1d687459","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"6705056e920e716ae011e88be80190ff07fdb61ca92476189e8def3a0a2f74d3","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011697a0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-lvqssm, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lvqssm\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-lvqssm, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lvqssm\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c41360\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-lvqssm\",\n },\n \u003c*fmt.wrapError | 0xc000b01060\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lvqssm\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011697a0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-lvqssm, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lvqssm\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-lvqssm, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lvqssm\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c41360\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-lvqssm\",\n },\n \u003c*fmt.wrapError | 0xc000b01060\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lvqssm\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"c2897c499d60d0604d77b4ca95d7a903e3a0346aeaa3999b053c70a69caf8294","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"e2554f79af5e0ee8cb5d93e6a151eb115622dd04f010ffcda2db9b562fd8a094","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f7a0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-97l6pl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000198900\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-97l6pl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000198700\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-97l6pl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00060d060\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-97l6pl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f7a0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-97l6pl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000198900\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-97l6pl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000198700\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-97l6pl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00060d060\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-97l6pl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"cf3d106c629cb723fcf74830d9ced23bc8584bc444e99b8dd8e887f45ed46792","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"2987f90b3b6254b0879d51079465b9075da8550df5f2deb6e1ba233776447a05","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001600cf0\u003e: \n failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-dt5mrz, error: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-dt5mrz\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-dt5mrz, error: failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dt5mrz\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014600d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-dt5mrz\",\n },\n \u003c*fmt.wrapError | 0xc001634d60\u003e{\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dt5mrz\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001600cf0\u003e: \n failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-dt5mrz, error: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-dt5mrz\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-dt5mrz, error: failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dt5mrz\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014600d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-dt5mrz\",\n },\n \u003c*fmt.wrapError | 0xc001634d60\u003e{\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-dt5mrz\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"d292f46ff7943e7961f4f4eb7ad5066796dccf706cbbd4714ebabe0b9208b254","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"de6be36e9374ec358dd2c2e74e9cc16a099378f3fff2b6fa8422060a54a1d7fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"8bf83a68de382b8d93782ab93fd777899c4633a4f1383fc6b72a378c7fb20626","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010385d0\u003e: \n failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bqppvw\" for cluster=\"ea-cluster\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bqppvw, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bqppvw\\\" for cluster=\\\"ea-cluster\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bqppvw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000fc8d0\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bqppvw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010385d0\u003e: \n failed waiting for external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-bqppvw\" for cluster=\"ea-cluster\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bqppvw, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-bqppvw\\\" for cluster=\\\"ea-cluster\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bqppvw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000fc8d0\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth external-auth in cluster ea-cluster in resource group external-auth-cluster-bqppvw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-sm9zjp\" for cluster=\"cidr-connectivity-test\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp, error: context deadline exceeded"}},{"row_id":"e28c5b58e5f9df968aa54de75c99f61f85c2e24fb59ff9c474fab11017aa80ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"e9ccd3ca9da737c461a675960f3feca425bb248ce9ac62e7f258af993224144a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"e526cb4b336c05120586f99464703ad4757975d7fadd18cb564a3adcee3b7a87","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00163a1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2jqm4s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007ad920\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2jqm4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ad900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2jqm4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ad8e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2jqm4s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00163a1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2jqm4s\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007ad920\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2jqm4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ad900\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2jqm4s\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007ad8e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2jqm4s\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"ece3f99e6db66790c7bae7adf890412110888a4722a07c42ff2cd66d466ce9bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"01032414f31ca7613f554efce1c370a96ba1195f41e0177874f075fd72cf9c5f","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011a41e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h7jb8n\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000bc7c20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-h7jb8n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000bc7c00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-h7jb8n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000bc7be0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-h7jb8n\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0011a41e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-h7jb8n\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000bc7c20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-h7jb8n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000bc7c00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-h7jb8n\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000bc7be0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-h7jb8n\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"effe434893e2341b81cc94e15ba1800228802100129f8d358b6d30232a94dfd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"33d2db0b95cb8769686c21dfe5ded90c62dfb064f9ecc30a2ca901aa88c4510d","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000da6e88\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-dvxxqx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001c19000\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-dvxxqx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001c18fe0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-dvxxqx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001c18fc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-dvxxqx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000da6e88\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-dvxxqx\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001c19000\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-dvxxqx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001c18fe0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-dvxxqx\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001c18fc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-dvxxqx\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"f054f9e94cb64281f6a94a51de806341797ac23f61809e27aac95427168cdf00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"ed14d14ba7aa60285af41a077efc0df8f380eec46e0adefcd7762214887fe2ac","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010b8318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-glcm9h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001507800\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-glcm9h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0015077e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-glcm9h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0015077c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-glcm9h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010b8318\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-glcm9h\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001507800\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-glcm9h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0015077e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-glcm9h\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0015077c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-glcm9h\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"f11d43b283e06e6ae540c605c1d09aa34839561c5bb78eb3d9efb35fdfd0871f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031466804123013120","occurred_at":"2026-03-10T20:26:43Z","failed_at":"e2e","signature_id":"fc589f2b65d16d02f15b4a4a2b7538d5cf14c420a94ee61fe4c352626e37f734","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001824210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-l4vjgt\" in resourcegroup=\"admin-credential-lifecycle-test-gddbct\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00091c8e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l4vjgt\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gddbct\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00091c8c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l4vjgt\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gddbct\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00091c8a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l4vjgt\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gddbct\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001824210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-l4vjgt\" in resourcegroup=\"admin-credential-lifecycle-test-gddbct\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00091c8e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l4vjgt\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gddbct\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00091c8c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l4vjgt\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gddbct\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00091c8a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-l4vjgt\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gddbct\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":19,"failure_pattern_ids":["4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":21,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4395/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458354898931712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4395,"pr_state":"closed","pr_sha":"5a9cad4cf159cb6512720c56c980044dd166fc76","final_merged_sha":"5a9cad4cf159cb6512720c56c980044dd166fc76","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T19:53:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4198,"pr_state":"open","pr_sha":"1b8d093b002bc026fd375fd63f1983eb3a0852e1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T19:51:46Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"4396f4d14c68dc0e4b0eb155737b1b9ce9fd51a579cee12e5eff977eb11d723d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"b4cbb06ce6a088183215914f79905936ac6f05592efc519ef803e70d4d5b37e5","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eb6528\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7smhwr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007e3ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-7smhwr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e3c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-7smhwr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e3c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-7smhwr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eb6528\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7smhwr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007e3ca0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-7smhwr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e3c80\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-7smhwr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0007e3c60\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-7smhwr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"447aea1a6ecaca752d0d54327a1ac4625dee2d30f749d457026b7b57883ed14d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"6336dc4868c9ac00dd6687bace733f1b6fea080dcada440d706247e42d231309","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001076768\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-khhpxr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d2c5e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-khhpxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d2c5c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-khhpxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d2c5a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-khhpxr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001076768\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-khhpxr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d2c5e0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-khhpxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d2c5c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-khhpxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d2c5a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-khhpxr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"4cd900dcaf7fdb92b28f527e5ca2ff4a37b4260d02852e9a02e191fe20f7f71c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"f31d3fb097d0632f39be5e42c51e1e6e10365a3235f4dab4c1c6e21ff935dfe5","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001792210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrckpfl\" in resourcegroup=\"ea-list-tqbdp4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009e8cc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listrckpfl\\\" in resourcegroup=\\\"ea-list-tqbdp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e8ca0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listrckpfl\\\" in resourcegroup=\\\"ea-list-tqbdp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e8a80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listrckpfl\\\" in resourcegroup=\\\"ea-list-tqbdp4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001792210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrckpfl\" in resourcegroup=\"ea-list-tqbdp4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009e8cc0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listrckpfl\\\" in resourcegroup=\\\"ea-list-tqbdp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e8ca0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listrckpfl\\\" in resourcegroup=\\\"ea-list-tqbdp4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009e8a80\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listrckpfl\\\" in resourcegroup=\\\"ea-list-tqbdp4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"670932384c01a71488d0a7470ee0f54cdebc5d815f7eeb8abcee0f22114519e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"6bc22d10fbd050830e1916c0fcc82a700c1613e37ba7151ce3e905ddc71cd5f8","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e942e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-shn2vs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00096f540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-shn2vs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096f520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-shn2vs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096f500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-shn2vs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e942e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-shn2vs\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00096f540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-shn2vs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096f520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-shn2vs\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00096f500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-shn2vs\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"7881c95bad74cc0e53b671239eb4d04ae18f5702a5426277b3dc0a2a94d6b49c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"395dac08395b6defbe648fc1cd727dcfb76aa1541c8864e5eeb49218728281e1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d46240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-pqbh6w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006fed60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pqbh6w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fed40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pqbh6w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fed20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pqbh6w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d46240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-pqbh6w\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0006fed60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pqbh6w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fed40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pqbh6w\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0006fed20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-pqbh6w\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"79f01ad5f18e310c14630cf9c7107b0d2a24382fac1ab19272b48baba71e8961","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"8839a121704502557efbc5f44db6fdaf0c1abc3c2d05cfe59a665f36234ff445","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00151a150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m5vhcc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000801020\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m5vhcc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000801000\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m5vhcc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000800fe0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m5vhcc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00151a150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m5vhcc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000801020\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m5vhcc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000801000\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m5vhcc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000800fe0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m5vhcc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"984ebd39cbec23bd297314f9400d759f58327cc9c4de62dd30ee414991472014","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"9c5839a0c8ce97744050fec10892a94d475048e6561fb13673558c659bf4b64c","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000efe468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tdcw2z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001015680\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tdcw2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001015660\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tdcw2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001015640\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tdcw2z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000efe468\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tdcw2z\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001015680\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tdcw2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001015660\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tdcw2z\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001015640\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-tdcw2z\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"a93955434b41432c026d19eb519da74758242c6414fa54ec71a65badbb219821","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"d8b169591d4d6970c01450355c3b2fe39f93ace192a9374aba01a74e7c214d6b","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00088a9d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6cxnq5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009011c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6cxnq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009011a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6cxnq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000901180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6cxnq5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00088a9d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6cxnq5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009011c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6cxnq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009011a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6cxnq5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000901180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6cxnq5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"be3b8107185c474ab8530eb3af4dec9363f09b7c1601855423d3f9321256205e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"c6247c503112df3c6cec5a6494c3eb391b43c143faaf73a57670910db88784d7","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:152]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d4810\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-6ggxsk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6ggxsk, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-6ggxsk\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6ggxsk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00094e200\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6ggxsk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:152]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d4810\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-6ggxsk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6ggxsk, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-6ggxsk\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6ggxsk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00094e200\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-6ggxsk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mcjzx6\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6, error: context deadline exceeded"}},{"row_id":"c25f59628908084304434346c40975fdfb02e5109a2996303b96f607badfa816","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"91451ae5097fc66b5343dc26898c1302e03c55f3c3f949b984ea41970743e5ca","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00134c1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-5t96v4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b19140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5t96v4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b19120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5t96v4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b19100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5t96v4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00134c1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-5t96v4\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b19140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5t96v4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b19120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5t96v4\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b19100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5t96v4\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"dd5500bee970c1fffa5d7096a35e0288a8794cb9c4d4310d2705623bcda013fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"d86c307f65002cc9ad3de9344fb74ea447668df46ba41158996828874a09102a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ef4648\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-tl4jh6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f221c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-tl4jh6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f221a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-tl4jh6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f22180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-tl4jh6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000ef4648\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-tl4jh6\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f221c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-tl4jh6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f221a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-tl4jh6\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f22180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-tl4jh6\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"f0d72b342c3ba9611f389371eae9b5130cb0406342e84104cf9399cbfff52366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031458012882800640","occurred_at":"2026-03-10T19:51:46Z","failed_at":"e2e","signature_id":"7c9f9be92117da0293e0ece90aebad0ba4205d8862eb5fc4bd197f8a93201f69","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eb6120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-gn8rhl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00072d380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gn8rhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00072d340\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gn8rhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00072d300\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gn8rhl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000eb6120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-gn8rhl\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00072d380\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gn8rhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00072d340\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gn8rhl\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00072d300\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-gn8rhl\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0650c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031457223149883392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"2aea2a82ad247d8ccb8de01037b74937ad277738","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T19:48:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031457183324966912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4394,"pr_state":"closed","pr_sha":"3aeb6faf2947e05a27cdf7a9d85809db6bb197be","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T19:48:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031446220454498304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4381,"pr_state":"closed","pr_sha":"9505bdbc5fdba631da63c7b5c2087fbc242619f3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T19:04:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6bb85d2025c8ca88e60a794256becc120508a31509e64b5fdff5928b7ff80cd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031446220454498304","occurred_at":"2026-03-10T19:04:56Z","failed_at":"provision","signature_id":"2b55142523fa9299ea35b86aab1017984080be00c907205107163447dbc59e2c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-10T19:55:34.522Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T19:55:34.846Z level=INFO msg=\"Source registry is a CI registry, using oc registry login\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image registry=registry.build02.ci.openshift.org\ntime=2026-03-10T19:55:34.995Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to get CI registry credentials via oc: oc registry login failed: info: Using registry public hostname registry.build02.ci.openshift.org\\nerror: reading JSON file \\\"/tmp/oc-auth-3352314194.json\\\": unmarshaling JSON at \\\"/tmp/oc-auth-3352314194.json\\\": unexpected end of JSON input\\n: exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"48bd3c1a3734653ee313cbfe49798293b8bc1ed7618d47b3bdc4d9bb21b7ab07","failure_pattern":"error: reading JSON file \"/tmp/oc-auth-3352314194.json\": unmarshaling JSON at \"/tmp/oc-auth-3352314194.json\": unexpected end of JSON input","search_query":"error: reading JSON file \\\"/tmp/oc-auth-3352314194.json\\\": unmarshaling JSON at \\\"/tmp/oc-auth-3352314194.json\\\": unexpected end of JSON input"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["48bd3c1a3734653ee313cbfe49798293b8bc1ed7618d47b3bdc4d9bb21b7ab07"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031441175570812928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3820,"pr_state":"closed","pr_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","final_merged_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T18:44:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031439049285177344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"1c23be7df06435684202d7c545112411912f987d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T18:36:26Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d96ab8c3ffcb02fe76f9d6c2499ca15bd16c110442fe83af5d9af8debbe33ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031439049285177344","occurred_at":"2026-03-10T18:36:26Z","failed_at":"provision","signature_id":"5ce21517de6a64a3dd83a21968f24222b0feabe6900765dd2f9858ac2cc35112","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-10T19:13:32.990Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-10T19:13:36.046Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-10T19:13:37.593Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ae8370847c9f196727dbb8b739d24301b9609659ed95d4a27f00945ecff16f8f\ntime=2026-03-10T19:43:32.726Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438617561272320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T18:34:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438374853677056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4383,"pr_state":"closed","pr_sha":"9128c3e9a3727d0be6602fb2b928876fd434f14c","final_merged_sha":"9128c3e9a3727d0be6602fb2b928876fd434f14c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T18:33:45Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"6e5198850faa64426e34bd80549e1f0b441b3cd432e9e66e75894219d52f998e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438374853677056","occurred_at":"2026-03-10T18:33:45Z","failed_at":"e2e","signature_id":"292a485b61a9fc430ff9f9ee3139676cba60a4e2f465ff9c3d25df9d600288c7","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000976620\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-77cmhg\" to finish creating: GET https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\",\n \"name\": \"9c94ce7e-df55-4d68-8fe3-b0f80110cfea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T19:27:23.96442781Z\",\n \"endTime\": \"2026-03-10T19:29:08.437592376Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj3677056.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:f8cd3561-101e-0059-4fc4-b09be1000000\\nTime:2026-03-10T19:28:51.1177193Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-77cmhg\\\" to finish creating: GET https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"name\\\": \\\"9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:27:23.96442781Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:29:08.437592376Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj3677056.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f8cd3561-101e-0059-4fc4-b09be1000000\\\\nTime:2026-03-10T19:28:51.1177193Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000976600\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-77cmhg\\\" to finish creating: GET https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"name\\\": \\\"9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:27:23.96442781Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:29:08.437592376Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj3677056.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f8cd3561-101e-0059-4fc4-b09be1000000\\\\nTime:2026-03-10T19:28:51.1177193Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.R...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000976620\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-77cmhg\" to finish creating: GET https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\",\n \"name\": \"9c94ce7e-df55-4d68-8fe3-b0f80110cfea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T19:27:23.96442781Z\",\n \"endTime\": \"2026-03-10T19:29:08.437592376Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj3677056.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:f8cd3561-101e-0059-4fc4-b09be1000000\\nTime:2026-03-10T19:28:51.1177193Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-77cmhg\\\" to finish creating: GET https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"name\\\": \\\"9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:27:23.96442781Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:29:08.437592376Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj3677056.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f8cd3561-101e-0059-4fc4-b09be1000000\\\\nTime:2026-03-10T19:28:51.1177193Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000976600\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-77cmhg\\\" to finish creating: GET https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"name\\\": \\\"9c94ce7e-df55-4d68-8fe3-b0f80110cfea\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:27:23.96442781Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:29:08.437592376Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj3677056.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f8cd3561-101e-0059-4fc4-b09be1000000\\\\nTime:2026-03-10T19:28:51.1177193Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.R...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"78412ab3a6b4fd5225e33c6b8cd3f0347dc2d56c32c494afde285d0650ac79ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438374853677056","occurred_at":"2026-03-10T18:33:45Z","failed_at":"e2e","signature_id":"12f06fe2c0ee9ff75ce58e24533d8c6f360185cd6ec4ec5a17f345022b8c9bcc","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0000df9e0\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.7:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.7:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc000746350\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.7:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0000df9e0\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.7:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.7:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc000746350\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.7:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error"}},{"row_id":"8b925a0fca546e48c5eaf9d9722b5cb865b29f1258e7c67e0c7270db221e9498","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438374853677056","occurred_at":"2026-03-10T18:33:45Z","failed_at":"e2e","signature_id":"bc414a3de8ca6ddad01b44fb37c141a37071ee4691f9821b4b75fb35c72d5155","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0012ab180\u003e: \n failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000fafb60\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 19:34:29 GMT\",\n ],\n \"Content-Length\": [\"316\"],\n \"X-Envoy-Upstream-Service-Time\": [\"25\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"f582fe12-33f9-4c9a-95d4-c934cb1441c8\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012ab0a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 118, 114, 106, 99, 122, 52, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 316,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j3677056.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredenti...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0012ab180\u003e: \n failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000fafb60\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 19:34:29 GMT\",\n ],\n \"Content-Length\": [\"316\"],\n \"X-Envoy-Upstream-Service-Time\": [\"25\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"f582fe12-33f9-4c9a-95d4-c934cb1441c8\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012ab0a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 118, 114, 106, 99, 122, 52, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 112, 97, 116, 99, 104, 45, 116, 97, 103, 115, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 316,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j3677056.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vrjcz4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredenti...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"8cca72ce7463e6c3f23527a732d9e1408187e14336f019a838554be3eac181fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438374853677056","occurred_at":"2026-03-10T18:33:45Z","failed_at":"e2e","signature_id":"e801c69d770fac27f50520c098d3a9c51a2854366e35c53370ef9e9e214b95ec","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a02c20\u003e: \n failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f47e60\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"82c53113-ce1b-4dd4-bd27-dd5b5ee92864\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 19:34:24 GMT\",\n ],\n \"Content-Length\": [\"346\"],\n \"X-Envoy-Upstream-Service-Time\": [\"27\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a02a80\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 100, 105, 115, 97, 98, 108, 101, 100, 45, 105, 109, 97, 103, 101, 45, 114, 101, 103, 105, 115, 116, 114, 121, 45, 122, 98, 116, 50, 109, 108, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 100, 105, 115, 97, 98, 108, 101, 100, 45, 105, 109, 97, 103, 101, 45, 114, 101, 103, 105, 115, 116, 114, 121, 45, 104, 99, 112, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 346,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j3677056.hcpsvc.osadev.cloud\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a02c20\u003e: \n failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-zbt2ml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f47e60\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"82c53113-ce1b-4dd4-bd27-dd5b5ee92864\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 19:34:24 GMT\",\n ],\n \"Content-Length\": [\"346\"],\n \"X-Envoy-Upstream-Service-Time\": [\"27\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a02a80\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 100, 105, 115, 97, 98, 108, 101, 100, 45, 105, 109, 97, 103, 101, 45, 114, 101, 103, 105, 115, 116, 114, 121, 45, 122, 98, 116, 50, 109, 108, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 100, 105, 115, 97, 98, 108, 101, 100, 45, 105, 109, 97, 103, 101, 45, 114, 101, 103, 105, 115, 116, 114, 121, 45, 104, 99, 112, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 346,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j3677056.hcpsvc.osadev.cloud\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"cf22f70ef9a3f97f39a967db6014e73762cc87875eae69120cb7427049266789","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031438374853677056","occurred_at":"2026-03-10T18:33:45Z","failed_at":"e2e","signature_id":"23c958520a68d3e6d879a943ae4be51e6c20548c1e350e507f3889f7f4bd1ba8","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a0b220\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4xs98c\": PUT https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4xs98c\\\": PUT https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a0b200\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4xs98c\\\": PUT https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000990c90\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"d89e5839-2ec1-4aaa-a7fe-70f6904cc07d\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 19:34:28 GMT\",\n ],\n \"Content-Length\": [\"340\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2101\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a0b120\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a0b220\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4xs98c\": PUT https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4xs98c\\\": PUT https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a0b200\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-4xs98c\\\": PUT https://rp.j3677056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-4xs98c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000990c90\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"d89e5839-2ec1-4aaa-a7fe-70f6904cc07d\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 19:34:28 GMT\",\n ],\n \"Content-Length\": [\"340\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2101\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a0b120\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4392/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031436425018216448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4392,"pr_state":"closed","pr_sha":"7ebfd60340924567770f9efc800ebac2076acc1e","final_merged_sha":"04b8269d6819dab9d5666de26e0179e4af64145a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T18:26:00Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4502264a616af4b1b1c69128003b07b0d75670d738c2d501de0846971c23c019","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4392/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031436425018216448","occurred_at":"2026-03-10T18:26:00Z","failed_at":"provision","signature_id":"6cbd97259ff63fdde668525ad9ca2d96963eed26da166ef18f6f847bfc03c07d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-10T19:20:11.600Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-10T19:20:12.198Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8216448.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8216448.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:066a9a53-0fdb-46be-ac16-f32bfa2af0ac databaseName:arohcpprow-rp-j8216448 databaseUrl:https://arohcpprow-rp-j8216448.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-6tqc6hj7/pipeline@sha256:3f5bf8fef96269f6b0b7fe4d229a38a31b921ec076789d23b904156eb1abb94d replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6tqc6hj7/pipeline:pull workloadIdentityClientId:c0b7e765-ba72-4ade-ba4f-7eb7f1c73b5f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:066a9a53-0fdb-46be-ac16-f32bfa2af0ac workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-10T19:20:12.198Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-10T19:20:12.441Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-10T19:20:12.740Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-10T19:20:12.740Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-10T19:20:12.850Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-10T19:25:19.095Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-10T19:25:19.095Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-10T19:25:19.095Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-10T19:25:22.577Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8216448.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8216448.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:066a9a53-0fdb-46be-ac16-f32bfa2af0ac databaseName:arohcpprow-rp-j8216448 databaseUrl:https://arohcpprow-rp-j8216448.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-6tqc6hj7/pipeline@sha256:3f5bf8fef96269f6b0b7fe4d229a38a31b921ec076789d23b904156eb1abb94d replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6tqc6hj7/pipeline:pull workloadIdentityClientId:c0b7e765-ba72-4ade-ba4f-7eb7f1c73b5f workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:066a9a53-0fdb-46be-ac16-f32bfa2af0ac workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-10T19:25:22.579Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQW8aMRCF7/srRnsCaR0BVSuFKoc07alKhZqeUvXgtV/BwetZjb0gIn58ZWjKFkHC9e1749nPfoVHIkHkTgwi3ZDVSSddewy4foJJX12w05jEhXlFe+mbbvBPwgphp8RWmxd5+LMgIiofICtncM/BJZayolILq4VpVa3NEsH2pLL6L3NrDHchZcN573f2yLKHthAFD5MchzPOTy5YF+a9kert4N91TuyuGiRxJp4I3XH47eb3uu2fpZ87gZIuJNdAmZ3l0uxZ92e0njcN9qTepntrZNZ5fwJF23mv6oN+HOzSgsU968xpxt6Zzc7lPa9fSLxy7gyQPAIhOaOPWF9A9AFGkGbCK2chd17H2F8+7m9JLbFRK935Po2y+PWxWHY1JCAhfskPNhZbWi8goHwZMemmpRppDQQaWJ2Q5UE5GU0+qNE7NR79GF9PJ6PpePJYDunqis563k/H14/lcFhs6YldoKUL9saFAOnVjAMd2tWv1XGfii21wvnrYdGKWrZ782VDKvI8r6hBjHqeR7JYCNWb3s9bRPMnAAD//9ypvxEMBAAA\"\ntime=2026-03-10T19:25:22.582Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-10T19:25:22.582Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031435806144466944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T18:23:33Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2107bf1f66e33c160d0e0ddcb7a062f4612176fd819b471d1e200630bf529365","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031435806144466944","occurred_at":"2026-03-10T18:23:33Z","failed_at":"e2e","signature_id":"b16fc01a21740a6c47d09e368b2e954c3bc3966beec9baedde18a1309d47e80b","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009d0140\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-96bfsn\" to finish creating: GET https://rp.j4466944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\",\n \"name\": \"1171ec57-e6db-40d0-a6ce-6dbca99f9632\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T19:50:09.79918139Z\",\n \"endTime\": \"2026-03-10T19:53:08.23159911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj4466944.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:2e42c44b-501e-0099-1dc7-b0ee07000000\\nTime:2026-03-10T19:52:47.7867575Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-96bfsn\\\" to finish creating: GET https://rp.j4466944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"name\\\": \\\"1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:50:09.79918139Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:53:08.23159911Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj4466944.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:2e42c44b-501e-0099-1dc7-b0ee07000000\\\\nTime:2026-03-10T19:52:47.7867575Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009d0120\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-96bfsn\\\" to finish creating: GET https://rp.j4466944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"name\\\": \\\"1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:50:09.79918139Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:53:08.23159911Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj4466944.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:2e42c44b-501e-0099-1dc7-b0ee07000000\\\\nTime:2026-03-10T19:52:47.7867575Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009d0140\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-96bfsn\" to finish creating: GET https://rp.j4466944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\",\n \"name\": \"1171ec57-e6db-40d0-a6ce-6dbca99f9632\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T19:50:09.79918139Z\",\n \"endTime\": \"2026-03-10T19:53:08.23159911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj4466944.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:2e42c44b-501e-0099-1dc7-b0ee07000000\\nTime:2026-03-10T19:52:47.7867575Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-96bfsn\\\" to finish creating: GET https://rp.j4466944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"name\\\": \\\"1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:50:09.79918139Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:53:08.23159911Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj4466944.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:2e42c44b-501e-0099-1dc7-b0ee07000000\\\\nTime:2026-03-10T19:52:47.7867575Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009d0120\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-96bfsn\\\" to finish creating: GET https://rp.j4466944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"name\\\": \\\"1171ec57-e6db-40d0-a6ce-6dbca99f9632\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T19:50:09.79918139Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T19:53:08.23159911Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj4466944.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:2e42c44b-501e-0099-1dc7-b0ee07000000\\\\nTime:2026-03-10T19:52:47.7867575Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031432487216877568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4391,"pr_state":"open","pr_sha":"dacf87647418e205a2b119537d656f914a585006","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T18:10:21Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"436a428d85b1e70554bf5b3888bf59cace63e5496c316ee59c1c76b4c33a1ab8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031432487216877568","occurred_at":"2026-03-10T18:10:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031431843621900288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4226,"pr_state":"closed","pr_sha":"b136b7d338c268e8e32f3d8709fbd32e07b07ee3","final_merged_sha":"138964dfc1d79a030eb9c07890259cb904e592a1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T18:07:48Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c8893158f15532b59a81a73c5613836f2959b4d1ac2d49ca7a0833be066d385f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031431843621900288","occurred_at":"2026-03-10T18:07:48Z","failed_at":"e2e","signature_id":"ef64222425fdce6e9af6ac14faa056312efce65919ddae4eb8e22417435094b8","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0000269c0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mcjzx6\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-mcjzx6\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a55ad0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0000269c0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mcjzx6\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-mcjzx6\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a55ad0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mcjzx6\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mcjzx6, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4319/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031419454927147008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4319,"pr_state":"closed","pr_sha":"244d96cb05aa29b95f158b2263feff5b21b68137","final_merged_sha":"244d96cb05aa29b95f158b2263feff5b21b68137","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T17:18:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031417974266531840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"830c5217e17a6f864742b271d997979658430036","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T17:12:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031412271296548864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3907,"pr_state":"closed","pr_sha":"4487e546850fc36ddfa989aa2a3500cc464111d8","final_merged_sha":"e9ca15da6d935a378fb165189c2441a771314f73","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T16:50:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031412056325885952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4205,"pr_state":"closed","pr_sha":"e430bc470c3354d54547e70972016250bdf9af38","final_merged_sha":"a36f37aae6ead63c15a10186a904f97e4e0fc50d","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T16:48:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031409720924835840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3820,"pr_state":"closed","pr_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","final_merged_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T16:39:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b9cf186c0a916209bd9e1ca557d29f5ca679a8399b78b15720945f24b6937bb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031409720924835840","occurred_at":"2026-03-10T16:39:45Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-10T17:09:20.033Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-10T17:09:22.741Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-10T17:09:24.001Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=252e7d91bcfd9798a43b366e8b47cccab07d0ba0b87fa05438e88bf174eb66d9\ntime=2026-03-10T17:10:54.772Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4835840/providers/Microsoft.Resources/deployments/252e7d91bcfd9798a43b366e8b47cccab07d0ba0b87fa05438e88bf174eb66d9/operationStatuses/08584284443221733016\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4835840/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4835840/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j4835840\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ac344776-a670-42b1-bee8-cfda1e896ad5:3/10/2026 5:10:44 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031401361081372672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"dca081b35c6754cb2894944ccbf1aff9365508c5","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T16:06:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031397707909697536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"e680d6c0bf7cbb898516f09eff2fab7a910876d4","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T15:52:00Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"467b399599b541384a9b51a745ab40bef6591e8fe1332595bc4146eab8e52470","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031397707909697536","occurred_at":"2026-03-10T15:52:00Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031396939018276864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4380,"pr_state":"closed","pr_sha":"69b2b212c38f98d8751e821a94b6221691685e4c","final_merged_sha":"69b2b212c38f98d8751e821a94b6221691685e4c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T15:49:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031396042183806976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T15:45:25Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"743dfdb17659dd4a3885f368bbe8cd92d5ec81f05f0b9d964987f038766cd550","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031396042183806976","occurred_at":"2026-03-10T15:45:25Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031395270360567808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4288,"pr_state":"closed","pr_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","final_merged_sha":"ee373fa7f2a4a9a9b5eaede1899ab3cfc444cf9d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T15:42:28Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b7702c929a1cf0dea4cf0076e149b5e6477066990ba8f07ba0c6da572335e325","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031395270360567808","occurred_at":"2026-03-10T15:42:28Z","failed_at":"provision","signature_id":"1e3c66f1b3aaa8e9531a0447e5fe758d4cfa54cfe1d96c0e23b03169e36088b8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"step graph","failure_text":"time=2026-03-10T16:33:07.282Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: Shell\\n Command: ./istio.sh\\n\"\ntime=2026-03-10T16:33:09.573Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256 -o istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ sha256sum -c istio-1.28.3-linux-amd64.tar.gz.sha256\\nsha256sum: istio-1.28.3-linux-amd64.tar.gz.sha256: no properly formatted SHA256 checksum lines found\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"00bf42e30ff9d333f122e52fc30a4967eab6684f10cfa8840e0bee5874b06083","failure_pattern":"error running Shell Step, failed to execute shell command: + set -euo pipefail + ISTIO_NAMESPACE=aks-istio-system + echo '********** Check istio is up and running **************' ********** Check istio is up and running ************** ++ kubectl get pod","search_query":"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256 -o istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ sha256sum -c istio-1.28.3-linux-amd64.tar.gz.sha256\\nsha256sum: istio-1.28.3-linux-amd64.tar.gz.sha256: no properly formatted SHA256 checksum lines found\\n exit status 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["00bf42e30ff9d333f122e52fc30a4967eab6684f10cfa8840e0bee5874b06083"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031395195144114176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3907,"pr_state":"closed","pr_sha":"4487e546850fc36ddfa989aa2a3500cc464111d8","final_merged_sha":"e9ca15da6d935a378fb165189c2441a771314f73","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T15:42:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d6a7d351ba5333d7b640761085775006d893748575abaa9bcbb19bfde827aa24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031395195144114176","occurred_at":"2026-03-10T15:42:10Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-10T16:19:05.157Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-10T16:19:06.686Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-10T16:19:08.195Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=193466c63e88ab1f67c893b50981287f131a4d95cda9d0c1bf16aaa9bbd6a676\ntime=2026-03-10T16:32:41.387Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4114176-svc/providers/Microsoft.Resources/deployments/193466c63e88ab1f67c893b50981287f131a4d95cda9d0c1bf16aaa9bbd6a676/operationStatuses/08584284473381261118\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4114176/providers/Microsoft.Resources/deployments/rp_cosmos_db\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"FailedIdentityOperation\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4114176/providers/Microsoft.DocumentDB/databaseAccounts/arohcpprow-rp-j4114176' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\\\\\\\\\\\\\"error\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"BadRequest\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4114176-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/admin-api' was not found.\\\\\\\\\\\\\\\"}}'.'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e7fca522141975e7bca69490a839ce35e4cd1ce022717bb9c58f7125b869988f","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/admin-api' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e7fca522141975e7bca69490a839ce35e4cd1ce022717bb9c58f7125b869988f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031391614097690624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4371,"pr_state":"closed","pr_sha":"11ecb878ec9a3501bb7ba58e755d71f6abf8de21","final_merged_sha":"e0b528c4354bf750dd12c0c682e19c640905138e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T15:27:56Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"65d1fbad7c70ee653ce850d00b97bc1a66e1f4bfffc9b1e88e8c6bd7985893d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031391614097690624","occurred_at":"2026-03-10T15:27:56Z","failed_at":"e2e","signature_id":"6a7311bfdd6cb42f5e474a2f6b2e40e7bd0bd6660be1179c3725c4448dd57b13","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000894420\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc00069a270\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000894420\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc00069a270\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4388/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031387359857086464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4388,"pr_state":"closed","pr_sha":"e884774cdad09b5a45a7e52a0929f97f5b7872d3","final_merged_sha":"e884774cdad09b5a45a7e52a0929f97f5b7872d3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-10T15:07:37Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1902f227d62b6981b35a698f2c13a176ab2c81d0664d96e07280299e6205c986","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4388/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031387359857086464","occurred_at":"2026-03-10T15:07:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031385039966572544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4383,"pr_state":"closed","pr_sha":"6cb8a515979049ef4a070c22702fd546ab3ca53d","final_merged_sha":"9128c3e9a3727d0be6602fb2b928876fd434f14c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T14:56:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031383825551986688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"4d70afb5e92810c3809c233c2365f388981f89a1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T14:42:05Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"04c375761ac311c8e66f0a1a681ef143504d6fc4dee4c2d129b8961677a90c02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031383825551986688","occurred_at":"2026-03-10T14:42:05Z","failed_at":"e2e","signature_id":"e892abd8c54799f382def9413016f6d6517066c809c3b50e9b02081403283155","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b751e0\u003e: \n failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00133aff0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:57 GMT\",\n ],\n \"Content-Length\": [\"327\"],\n \"X-Envoy-Upstream-Service-Time\": [\"27\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"eba75de9-6f77-49a9-8261-41301d1686fe\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b750a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 114, 103, 45, 99, 108, 117, 115, 116, 101, 114, 45, 98, 97, 99, 107, 45, 118, 101, 114, 115, 105, 111, 110, 45, 53, 119, 104, 113, 102, 54, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 99, 108, 117, 115, 116, 101, 114, 45, 118, 101, 114, 45, 52, 45, 49, 57, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 327,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j1986688.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b751e0\u003e: \n failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00133aff0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:57 GMT\",\n ],\n \"Content-Length\": [\"327\"],\n \"X-Envoy-Upstream-Service-Time\": [\"27\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"eba75de9-6f77-49a9-8261-41301d1686fe\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b750a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 114, 103, 45, 99, 108, 117, 115, 116, 101, 114, 45, 98, 97, 99, 107, 45, 118, 101, 114, 115, 105, 111, 110, 45, 53, 119, 104, 113, 102, 54, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 99, 108, 117, 115, 116, 101, 114, 45, 118, 101, 114, 45, 52, 45, 49, 57, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 327,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j1986688.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-5whqf6/providers/...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"837eaa9a5b581faf78c69d19f8657ded569123c5e9f9655121429b86d4381ec3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031383825551986688","occurred_at":"2026-03-10T14:42:05Z","failed_at":"e2e","signature_id":"c356cc77fc5370978b5aa68a77981ce82709c1f7d303cb195a1ce7e5a232904e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:173]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00141e7a0\u003e: \n failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0008b7830\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"032dd7ef-e3c7-4688-a613-f8a1b5d25b8e\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:48 GMT\",\n ],\n \"Content-Length\": [\"346\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1028\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00141e6a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 100, 109, 105, 110, 45, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 45, 108, 105, 102, 101, 99, 121, 99, 108, 101, 45, 116, 101, 115, 116, 45, 104, 120, 106, 107, 56, 109, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 97, 100, 109, 105, 110, 45, 99, 114, 101, 100, 45, 108, 105, 102, 101, 99, 121, 99, 108, 101, 45, 116, 57, 53, 54, 104, 102, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 346,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j1986688.hcpsvc.osadev.cloud\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:173]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00141e7a0\u003e: \n failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hxjk8m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-t956hf\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0008b7830\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"032dd7ef-e3c7-4688-a613-f8a1b5d25b8e\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:48 GMT\",\n ],\n \"Content-Length\": [\"346\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1028\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00141e6a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 100, 109, 105, 110, 45, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 45, 108, 105, 102, 101, 99, 121, 99, 108, 101, 45, 116, 101, 115, 116, 45, 104, 120, 106, 107, 56, 109, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 97, 100, 109, 105, 110, 45, 99, 114, 101, 100, 45, 108, 105, 102, 101, 99, 121, 99, 108, 101, 45, 116, 57, 53, 54, 104, 102, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 346,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j1986688.hcpsvc.osadev.cloud\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"9fdc6d85cc8dc5ee203da4c3a8b21732af5c2718b5dd6356d471144f38c5987e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031383825551986688","occurred_at":"2026-03-10T14:42:05Z","failed_at":"e2e","signature_id":"d9ff33b180dfef1d70b80ce9c41c6e26800db2cb4e81f7f43b9bf81a21b568ed","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:214]: Timed out after 300.003s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b170c0\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:214]: Timed out after 300.003s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b170c0\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3","failure_pattern":"source \"fake-source2.example.com/fake\" not found in any IDMS","search_query":"source \"fake-source2.example.com/fake\" not found in any IDMS"}},{"row_id":"ab0b5c8add204de1309517c7633baa4faf48c676802aba05abe18770ffcf6050","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031383825551986688","occurred_at":"2026-03-10T14:42:05Z","failed_at":"e2e","signature_id":"6c9c485ea4dc5aaea7ded971e214db7d297c8e28d63fa7c193cd4b18328268f1","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000775300\u003e: \n failed to create NodePool nodepool-128GiB: failed starting nodepool creation \"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-g7rf7z\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool nodepool-128GiB: failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-g7rf7z\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007752e0\u003e{\n msg: \"failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-g7rf7z\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011b5da0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:57 GMT\",\n ],\n \"Content-Length\": [\"344\"],\n \"X-Envoy-Upstream-Service-Time\": [\"85\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"7e936490-be89-47eb-a302-7b10d5507757\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000775200\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000775300\u003e: \n failed to create NodePool nodepool-128GiB: failed starting nodepool creation \"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-g7rf7z\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool nodepool-128GiB: failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-g7rf7z\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007752e0\u003e{\n msg: \"failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-g7rf7z\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-g7rf7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011b5da0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:57 GMT\",\n ],\n \"Content-Length\": [\"344\"],\n \"X-Envoy-Upstream-Service-Time\": [\"85\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"7e936490-be89-47eb-a302-7b10d5507757\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000775200\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"ec957c7fc2de220e83e8676097801277c413d2f1548eaa415dd1843a5d84509c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031383825551986688","occurred_at":"2026-03-10T14:42:05Z","failed_at":"e2e","signature_id":"729ca4bad24c846d523e7ee1d3c62d0aa163d0cf235da6ab7c2576c43b784514","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a3afe0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-g779ch\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-g779ch\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a3afc0\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-g779ch\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012bad50\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"3707dcaf-af3a-4d6e-bc43-be2c2e5217f8\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:56 GMT\",\n ],\n \"Content-Length\": [\"345\"],\n \"X-Envoy-Upstream-Service-Time\": [\"103\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a3aee0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a3afe0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-g779ch\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-g779ch\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a3afc0\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-g779ch\\\": PUT https://rp.j1986688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g779ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012bad50\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"3707dcaf-af3a-4d6e-bc43-be2c2e5217f8\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 15:57:56 GMT\",\n ],\n \"Content-Length\": [\"345\"],\n \"X-Envoy-Upstream-Service-Time\": [\"103\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a3aee0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031374655939416064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"eac14dfe2d1e1037f83e2effb8b3fabd24958599","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T14:09:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ba3e8d7fb3f71798f5cc71d809c616b635faf089e4a9882f54b67c164c700f92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031374655939416064","occurred_at":"2026-03-10T14:09:07Z","failed_at":"e2e","signature_id":"f1ce9448369d3f03665cbf260a11e1c323106816315aec02497108391c23d55d","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:244]: Unexpected error:\n \u003c*errors.joinError | 0xc001362360\u003e: \n cluster listing in non-existent resource group: expected error but none occurred\n node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-jjgrg4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*errors.errorString | 0xc001142210\u003e{\n s: \"cluster listing in non-existent resource group: expected error but none occurred\",\n },\n \u003c*errors.errorString | 0xc0011431a0\u003e{\n s: \"node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-jjgrg4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n \u003c*errors.errorString | 0xc000f366a0\u003e{\n s: \"node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:244]: Unexpected error:\n \u003c*errors.joinError | 0xc001362360\u003e: \n cluster listing in non-existent resource group: expected error but none occurred\n node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-jjgrg4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*errors.errorString | 0xc001142210\u003e{\n s: \"cluster listing in non-existent resource group: expected error but none occurred\",\n },\n \u003c*errors.errorString | 0xc0011431a0\u003e{\n s: \"node pool listing in RG with no clusters: expected error containing keywords 'parent resource not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-jjgrg4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n \u003c*errors.errorString | 0xc000f366a0\u003e{\n s: \"node pool listing in non-existent resource group: expected error containing keywords 'resource group not found', got: GET https://rp.j9416064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/non-existent-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["223928d9cb18511fa83ff41f30aabddf9e3d125665ad0fdad9bb0068231d5de6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031373965540200448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"eac14dfe2d1e1037f83e2effb8b3fabd24958599","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T14:04:05Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"137ca93c43b374031ebaf50d230dd6df6f900ca26a0e306b01c818e88f513f82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031373965540200448","occurred_at":"2026-03-10T14:04:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"1c23be7df06435684202d7c545112411912f987d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T13:59:50Z"},"failed_at":["e2e"],"failed_test_count":21,"occurrences":[{"row_id":"0493c5d08cac5a28818f19e163d881fab1dc9320c53f8c4ed6f563724888e5b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"34ca2dbaf9213ace3b2b679361c2ffbcf5f067850842926e805121509a5480c8","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0000fe228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-k726vg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d9b820\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-k726vg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d9b800\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-k726vg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d9b7e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-k726vg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0000fe228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-k726vg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d9b820\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-k726vg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d9b800\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-k726vg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d9b7e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-k726vg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"16e9a1a80c77821b70f06de6ca67f8db28be53a3428ef84e461e08e597321b3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"83d0dc520bebbce30bb7a04a5ed289cc04a259c4491c876a6203652c80b93645","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b9740\u003e: \n failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-lq8w22, error: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lq8w22\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-lq8w22, error: failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-lq8w22\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000166850\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-lq8w22\",\n },\n \u003c*fmt.wrapError | 0xc001000ee0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-lq8w22\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b9740\u003e: \n failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-lq8w22, error: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lq8w22\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-lq8w22, error: failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-lq8w22\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000166850\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-lq8w22\",\n },\n \u003c*fmt.wrapError | 0xc001000ee0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-lq8w22\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"2ebbc854c9177387d1899db5510982f7b1b2651a79471cbb1420ae137bf1985b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"784bc7da7239da9dfa6c79b400801368c1b1875e0baa86fc3c5f5e7e9d5c406f","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fdc2b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-sq6hmg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012e8fe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-sq6hmg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012e8fc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-sq6hmg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012e8fa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-sq6hmg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fdc2b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-sq6hmg\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012e8fe0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-sq6hmg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012e8fc0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-sq6hmg\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012e8fa0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-sq6hmg\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"397a47c3ee6862ab2c324b12c664102ad7e376c2ea437341ec66f99f5558b16b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"9ace4b5a4843ea9fe7cfb3dae28fd16224575b03efafb497d6e0a31b24414791","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e82408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wdq7k7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00052b6a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wdq7k7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00052b680\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wdq7k7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00052b660\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wdq7k7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000e82408\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wdq7k7\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00052b6a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wdq7k7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00052b680\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wdq7k7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00052b660\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wdq7k7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"3b1da5554cb135e701cad3c0a1af9c862117bfddc7c340ec2d9aaa0d03ee134f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"33302b88f21cc83fa4072904dfb190eba7d8fda866d51e4dc2cb832928cb8a1e","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"s report:\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled (Spec Runtime: 1h29m59.624s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n In [It] (Node Runtime: 1h29m59.623s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n At [By Step] creating the HCP cluster (Step Runtime: 36m8.55s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:76\n\n Spec Goroutine\n goroutine 37 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc0012e8460}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc0012e8460}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0001aca80?}, {{0x824f9c8?, 0xc0004c0000?}, 0xc00063f340?}, 0xc00009b7e0, {0xc0014343c0, 0x15}, {0x7841a74, 0x14}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func23.2({0x7fbcf0684460, 0xc00066a2a0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:77\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(ctx,\n | \tGinkgoLogr,\n | \t*resourceGroup.Name,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc00066a2a0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 81\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83 @ 03/10/26 16:42:11.169\n \"ts\"=\"2026-03-10 16:42:11.169810\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.170007\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-5k975s\" \"resourceGroup\"=\"np-autoscaling-cb9l84\"\n \"ts\"=\"2026-03-10 16:42:12.605524\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-5k975s\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.934752\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.934928\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.935067\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.938167\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-146\"\n \"ts\"=\"2026-03-10 16:42:13.938214\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-cb9l84\"\n \"ts\"=\"2026-03-10 16:42:14.762169\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-146\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:15.414810\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-cb9l84\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-03-10 16:42:16.145633\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-146\" \"deployment\"=\"msi-deployment-146\"\n \"ts\"=\"2026-03-10 16:42:16.860089\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-cb9l84\" \"deployment\"=\"customer-infra-np-autoscale-cluster-5k975s\"\n \"ts\"=\"2026-03-10 16:42:18.287841\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.288071\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.288315\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-cb9l84\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ns report:\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled (Spec Runtime: 1h29m59.624s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n In [It] (Node Runtime: 1h29m59.623s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n At [By Step] creating the HCP cluster (Step Runtime: 36m8.55s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:76\n\n Spec Goroutine\n goroutine 37 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc0012e8460}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc0012e8460}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0001aca80?}, {{0x824f9c8?, 0xc0004c0000?}, 0xc00063f340?}, 0xc00009b7e0, {0xc0014343c0, 0x15}, {0x7841a74, 0x14}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func23.2({0x7fbcf0684460, 0xc00066a2a0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:77\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(ctx,\n | \tGinkgoLogr,\n | \t*resourceGroup.Name,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc00066a2a0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 81\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83 @ 03/10/26 16:42:11.169\n \"ts\"=\"2026-03-10 16:42:11.169810\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.170007\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-5k975s\" \"resourceGroup\"=\"np-autoscaling-cb9l84\"\n \"ts\"=\"2026-03-10 16:42:12.605524\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-5k975s\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.934752\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.934928\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.935067\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.938167\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-146\"\n \"ts\"=\"2026-03-10 16:42:13.938214\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-cb9l84\"\n \"ts\"=\"2026-03-10 16:42:14.762169\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-146\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:15.414810\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-cb9l84\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-03-10 16:42:16.145633\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-146\" \"deployment\"=\"msi-deployment-146\"\n \"ts\"=\"2026-03-10 16:42:16.860089\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-cb9l84\" \"deployment\"=\"customer-infra-np-autoscale-cluster-5k975s\"\n \"ts\"=\"2026-03-10 16:42:18.287841\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.288071\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.288315\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-cb9l84\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"4a32f8ded6ae5f65fd42b95ffac315e3dfe490cd3342553508eb57c0a9ef97a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"97f5e0076a90e4379281acd79684513c55fa305c0d0b567d0b712be49c18ae0d","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fb62b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n5g8lq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000749e40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n5g8lq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000749e20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n5g8lq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000749e00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n5g8lq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fb62b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n5g8lq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000749e40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n5g8lq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000749e20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n5g8lq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000749e00\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n5g8lq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"4ce06d3d3d5da1ba3b86fac7a5efd2047beb9f364ec41d0a2ef80273ac8365be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"440eab1c79c62bfb3ac643258443e569a41fbc807fbcf9c8e6335d3d068944b5","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00076e2a0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-rxmv5f, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rxmv5f\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-rxmv5f, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rxmv5f\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a16280\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-rxmv5f\",\n },\n \u003c*fmt.wrapError | 0xc000c56260\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rxmv5f\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00076e2a0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-rxmv5f, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rxmv5f\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-rxmv5f, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rxmv5f\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a16280\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-rxmv5f\",\n },\n \u003c*fmt.wrapError | 0xc000c56260\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-rxmv5f\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"4e4c55bcb207965b547489efe500cbf92b0043456c7b0c3f9c6816111a9ab361","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"40d39e52c3d7fb8e66cca00d6ccf9a2d6495b2f7243ed5a28f1d1076c616a381","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001580258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qxsk8f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000910a40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qxsk8f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000910a20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qxsk8f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009109e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qxsk8f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001580258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-qxsk8f\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000910a40\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qxsk8f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000910a20\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qxsk8f\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009109e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-qxsk8f\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"5172e0d635f3052f8b5e450391effd7c2e1b071f2787de7f99e8a3629f8d2495","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"c74e84fe1025162fb949ae1958a03a42c4a42173feec627e2103411ec6ef4ccd","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015802e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6lnzw5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012ba200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6lnzw5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012ba1c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6lnzw5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012ba1a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6lnzw5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015802e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6lnzw5\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0012ba200\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6lnzw5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012ba1c0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6lnzw5\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0012ba1a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6lnzw5\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"52a509f5a6bd135390a6e07dcfdb5ae6027b7579a82bf487d80aea16b90407c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"d3647878e150901b28ee6d84c4797358f5d46b4a5fc91b8ff8faba021ae2b5c4","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Engineering should be able to retrieve kusto logs for a cluster and services (Spec Runtime: 1h29m59.619s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n In [It] (Node Runtime: 1h29m59.619s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n At [By Step] creating the HCP cluster (Step Runtime: 35m58.743s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:79\n\n Spec Goroutine\n goroutine 108 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000c484b0}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000c484b0}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0008903f0?}, {{0x824f9c8?, 0xc000a3ec00?}, 0xc001690258?}, 0xc00009b808, {0xc000082900, 0x11}, {0x7850b87, 0x17}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func22.2({0x7f6280596a60, 0xc0009642d0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:80\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc0009642d0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 105\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87 @ 03/10/26 16:42:11.152\n \"ts\"=\"2026-03-10 16:42:11.152421\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.152597\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-9tns56\" \"resourceGroup\"=\"kusto-logs-4rrctc\"\n \"ts\"=\"2026-03-10 16:42:13.106436\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-engineering-hcp-cluster-9tns56\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.813634\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.813756\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.813850\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.815776\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-138\"\n \"ts\"=\"2026-03-10 16:42:13.815811\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"kusto-logs-4rrctc\"\n \"ts\"=\"2026-03-10 16:42:14.673064\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-138\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:15.383665\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-4rrctc\" \"deployment\"=\"pooledNonMsiScopedAssignments-5borhjiq7mwd4\"\n \"ts\"=\"2026-03-10 16:42:16.051229\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-138\" \"deployment\"=\"msi-deployment-138\"\n \"ts\"=\"2026-03-10 16:42:16.728183\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-4rrctc\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-9tns56\"\n \"ts\"=\"2026-03-10 16:42:18.078393\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.078613\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.078797\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"kusto-logs-4rrctc\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nspecs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Engineering should be able to retrieve kusto logs for a cluster and services (Spec Runtime: 1h29m59.619s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n In [It] (Node Runtime: 1h29m59.619s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n At [By Step] creating the HCP cluster (Step Runtime: 35m58.743s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:79\n\n Spec Goroutine\n goroutine 108 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000c484b0}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000c484b0}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0008903f0?}, {{0x824f9c8?, 0xc000a3ec00?}, 0xc001690258?}, 0xc00009b808, {0xc000082900, 0x11}, {0x7850b87, 0x17}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func22.2({0x7f6280596a60, 0xc0009642d0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:80\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc0009642d0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 105\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87 @ 03/10/26 16:42:11.152\n \"ts\"=\"2026-03-10 16:42:11.152421\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.152597\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-9tns56\" \"resourceGroup\"=\"kusto-logs-4rrctc\"\n \"ts\"=\"2026-03-10 16:42:13.106436\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-engineering-hcp-cluster-9tns56\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.813634\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.813756\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-fb6d207289fe1114ce672abf5865312a466e24e7ab0cc85e59e941314773e2c8.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.813850\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.815776\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-138\"\n \"ts\"=\"2026-03-10 16:42:13.815811\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"kusto-logs-4rrctc\"\n \"ts\"=\"2026-03-10 16:42:14.673064\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-138\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:15.383665\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-4rrctc\" \"deployment\"=\"pooledNonMsiScopedAssignments-5borhjiq7mwd4\"\n \"ts\"=\"2026-03-10 16:42:16.051229\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-138\" \"deployment\"=\"msi-deployment-138\"\n \"ts\"=\"2026-03-10 16:42:16.728183\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"kusto-logs-4rrctc\" \"deployment\"=\"customer-infra-engineering-hcp-cluster-9tns56\"\n \"ts\"=\"2026-03-10 16:42:18.078393\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.078613\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.078797\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"kusto-logs-4rrctc\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"70fdb14d18963bf084a157c8d714d6861a1922f3963c9e9e44c5fd58cb775b2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"360a042ef4a20283a4015c518d9070b1365a672c4f1b32e3141b328595c7b6a4","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000810d38\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00059cd20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kkw9j2\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4vxjb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059cd00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kkw9j2\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4vxjb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059cce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kkw9j2\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4vxjb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000810d38\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00059cd20\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kkw9j2\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4vxjb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059cd00\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kkw9j2\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4vxjb\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00059cce0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kkw9j2\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4vxjb\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"78595c3da2988d66e3ab981ec12b9a62e5aa9eb7557e816947a46f61a44608d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"59975f704fa16478973dd49ebba45ed7245927dbb5aa7e8ce101267af083dd55","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001044240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cg4txq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009bf640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cg4txq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009bf620\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cg4txq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009bf600\u003e{\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cg4txq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001044240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cg4txq\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009bf640\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cg4txq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009bf620\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cg4txq\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0009bf600\u003e{\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cg4txq\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"821537668a1fd79f9315d9ce88c0dd4677a63b1b8f2d6239e844ece626a36a35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"841cce46b8d93aa419b8dd2db39b8310ebc700a87519152ef0bcb571985a3a9a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:120]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010beed0\u003e: \n failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-brbm4q, error: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-brbm4q\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-brbm4q, error: failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-brbm4q\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011da740\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-brbm4q\",\n },\n \u003c*fmt.wrapError | 0xc00051a800\u003e{\n msg: \"failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-brbm4q\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:120]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010beed0\u003e: \n failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-brbm4q, error: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-brbm4q\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-brbm4q, error: failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-brbm4q\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011da740\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-brbm4q\",\n },\n \u003c*fmt.wrapError | 0xc00051a800\u003e{\n msg: \"failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-brbm4q\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"8cdd2f5f55c6bcbf61b86ff7b9ff3b46336fc7641e094a86b56e01a4ee87e015","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"246d3cd4065793516993041542ae0c30fc4749a566aa067806d08cf48694433b","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001304138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hhc27d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000cff140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-hhc27d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cff120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-hhc27d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cff100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-hhc27d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001304138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-hhc27d\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000cff140\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-hhc27d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cff120\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-hhc27d\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000cff100\u003e{\n msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-hhc27d\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"b76a8ec956eb36c9de007c904db62bc14f4734b72999f402ad3f98a4ba8bd7ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"36756c6a072955a44053fbbf489e89913ad4675d27ac21fbde300a83201f5708","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request (Spec Runtime: 1h29m59.547s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:120\n In [It] (Node Runtime: 1h29m59.547s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:120\n At [By Step] creating the HCP cluster (Step Runtime: 33m59.309s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:152\n\n Spec Goroutine\n goroutine 30 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000032730}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000032730}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc00039b810?}, {{0x824f9c8?, 0xc000736000?}, 0xc000979f90?}, 0xc00009b798, {0xc00150a0a8, 0x11}, {0x783722d, 0x12}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func13.2.1({0x7f4a7442c3c8, 0xc0008f8390})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:153\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc0008f8390?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160 @ 03/10/26 16:42:11.159\n \"ts\"=\"2026-03-10 16:42:11.159834\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.160053\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-tqzd7z\" \"resourceGroup\"=\"patch-tags-gz26w9\"\n \"ts\"=\"2026-03-10 16:42:12.625579\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-tqzd7z\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.498474\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.498612\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.498707\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.500587\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-66\"\n \"ts\"=\"2026-03-10 16:42:13.500631\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-gz26w9\"\n \"ts\"=\"2026-03-10 16:42:14.310344\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-66\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:14.987312\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-gz26w9\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n \"ts\"=\"2026-03-10 16:42:15.710879\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-66\" \"deployment\"=\"msi-deployment-66\"\n \"ts\"=\"2026-03-10 16:42:16.375074\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-gz26w9\" \"deployment\"=\"customer-infra-patch-tags-cluster-tqzd7z\"\n \"ts\"=\"2026-03-10 16:42:17.814778\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:17.815089\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:17.815329\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-gz26w9\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nskip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request (Spec Runtime: 1h29m59.547s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:120\n In [It] (Node Runtime: 1h29m59.547s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:120\n At [By Step] creating the HCP cluster (Step Runtime: 33m59.309s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:152\n\n Spec Goroutine\n goroutine 30 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000032730}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000032730}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc00039b810?}, {{0x824f9c8?, 0xc000736000?}, 0xc000979f90?}, 0xc00009b798, {0xc00150a0a8, 0x11}, {0x783722d, 0x12}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func13.2.1({0x7f4a7442c3c8, 0xc0008f8390})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:153\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc0008f8390?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160 @ 03/10/26 16:42:11.159\n \"ts\"=\"2026-03-10 16:42:11.159834\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.160053\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-tqzd7z\" \"resourceGroup\"=\"patch-tags-gz26w9\"\n \"ts\"=\"2026-03-10 16:42:12.625579\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-tqzd7z\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.498474\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.498612\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.498707\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.500587\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-66\"\n \"ts\"=\"2026-03-10 16:42:13.500631\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-gz26w9\"\n \"ts\"=\"2026-03-10 16:42:14.310344\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-66\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:14.987312\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-gz26w9\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n \"ts\"=\"2026-03-10 16:42:15.710879\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-66\" \"deployment\"=\"msi-deployment-66\"\n \"ts\"=\"2026-03-10 16:42:16.375074\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-gz26w9\" \"deployment\"=\"customer-infra-patch-tags-cluster-tqzd7z\"\n \"ts\"=\"2026-03-10 16:42:17.814778\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:17.815089\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:17.815329\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-gz26w9\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"c63833e0406afd370db99d183b964066b4eb153868de40b2dfcb61bdde5716ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"77a31c1378200aa8e551994c018190007b9e7e932069ca0cd91127e86ad61d64","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"ip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to create node pools with ARM64-based VMs (Spec Runtime: 1h29m59.701s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n In [It] (Node Runtime: 1h29m59.701s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n At [By Step] creating the HCP cluster (Step Runtime: 35m59.6s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:82\n\n Spec Goroutine\n goroutine 106 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000a34aa0}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000a34aa0}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0004cebd0?}, {{0x824f9c8?, 0xc000668980?}, 0xc0000833c8?}, 0xc000f957e8, {0xc000d56090, 0x17}, {0x7841948, 0x14}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func4.2({0x7feeb4458260, 0xc000ce6a20})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:83\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc000ce6a20?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90 @ 03/10/26 16:42:11.165\n \"ts\"=\"2026-03-10 16:42:11.165938\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.166147\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-dvlvjr\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\"\n \"ts\"=\"2026-03-10 16:42:13.109545\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-dvlvjr\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.675589\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.675720\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.675847\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.678043\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-99\"\n \"ts\"=\"2026-03-10 16:42:13.678091\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\"\n \"ts\"=\"2026-03-10 16:42:14.543605\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-99\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:15.262106\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n \"ts\"=\"2026-03-10 16:42:16.006348\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-99\" \"deployment\"=\"msi-deployment-99\"\n \"ts\"=\"2026-03-10 16:42:16.694697\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-dvlvjr\"\n \"ts\"=\"2026-03-10 16:42:18.161866\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.162091\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.162364\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to create node pools with ARM64-based VMs (Spec Runtime: 1h29m59.701s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n In [It] (Node Runtime: 1h29m59.701s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n At [By Step] creating the HCP cluster (Step Runtime: 35m59.6s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:82\n\n Spec Goroutine\n goroutine 106 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000a34aa0}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000a34aa0}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0004cebd0?}, {{0x824f9c8?, 0xc000668980?}, 0xc0000833c8?}, 0xc000f957e8, {0xc000d56090, 0x17}, {0x7841948, 0x14}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func4.2({0x7feeb4458260, 0xc000ce6a20})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:83\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc000ce6a20?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90 @ 03/10/26 16:42:11.165\n \"ts\"=\"2026-03-10 16:42:11.165938\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.166147\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-dvlvjr\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\"\n \"ts\"=\"2026-03-10 16:42:13.109545\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-dvlvjr\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.675589\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.675720\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.675847\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.678043\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-99\"\n \"ts\"=\"2026-03-10 16:42:13.678091\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\"\n \"ts\"=\"2026-03-10 16:42:14.543605\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-99\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:15.262106\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n \"ts\"=\"2026-03-10 16:42:16.006348\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-99\" \"deployment\"=\"msi-deployment-99\"\n \"ts\"=\"2026-03-10 16:42:16.694697\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-dvlvjr\"\n \"ts\"=\"2026-03-10 16:42:18.161866\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.162091\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.162364\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-zqzxbn\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d558e322594cfeb87f652fc40903ba66b344a7077c9d33e6e2bc3e1593ead982","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"87f38f58bc88185bfc452574241bcee71e8d69904f6cf52c01f143517180a3c0","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012f81e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-f79m7k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d18460\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-f79m7k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d18440\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-f79m7k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fd94a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-f79m7k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0012f81e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-f79m7k\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d18460\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-f79m7k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d18440\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-f79m7k\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000fd94a0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-f79m7k\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"eaa24df89a491e1b2b4b9634bf22cf0248eee20acba011ff993785918ef07bcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"49d81e218e96a597d49e95fe811e9688d50b665bf242f38085b5096d34b7c2ef","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011def00\u003e: \n failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-sm9zjp\" for cluster=\"cidr-connectivity-test\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"ext-auth-cidr\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sm9zjp\\\" for cluster=\\\"cidr-connectivity-test\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00095cd20\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011def00\u003e: \n failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-sm9zjp\" for cluster=\"cidr-connectivity-test\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"ext-auth-cidr\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sm9zjp\\\" for cluster=\\\"cidr-connectivity-test\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00095cd20\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-sm9zjp\" for cluster=\"cidr-connectivity-test\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sm9zjp, error: context deadline exceeded"}},{"row_id":"efea9f3c48e5fb7f1dccd0380d99ffc1c72e818cec0b257d090e7c50ff5957fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"3c7c008a14ee0927ba89cf59395979901a6a74e94d863d42b14382ccc189660f","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010506f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-tq274r\" in resourcegroup=\"cluster-listing-zjw4z7\" to finish deleting: context canceled\n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-wdkbpb\" in resourcegroup=\"cluster-listing-mm6jmt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f87b80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-tq274r\\\" in resourcegroup=\\\"cluster-listing-zjw4z7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87b60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-tq274r\\\" in resourcegroup=\\\"cluster-listing-zjw4z7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87b40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-tq274r\\\" in resourcegroup=\\\"cluster-listing-zjw4z7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000f87c00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-wdkbpb\\\" in resourcegroup=\\\"cluster-listing-mm6jmt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87be0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-wdkbpb\\\" in resourcegroup=\\\"cluster-listing-mm6jmt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87bc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-wdkbpb\\\" in resourcegroup=\\\"cluster-listing-mm6jmt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0010506f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-tq274r\" in resourcegroup=\"cluster-listing-zjw4z7\" to finish deleting: context canceled\n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-wdkbpb\" in resourcegroup=\"cluster-listing-mm6jmt\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f87b80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-tq274r\\\" in resourcegroup=\\\"cluster-listing-zjw4z7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87b60\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-tq274r\\\" in resourcegroup=\\\"cluster-listing-zjw4z7\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87b40\u003e{\n msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-tq274r\\\" in resourcegroup=\\\"cluster-listing-zjw4z7\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xc000f87c00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-wdkbpb\\\" in resourcegroup=\\\"cluster-listing-mm6jmt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87be0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-wdkbpb\\\" in resourcegroup=\\\"cluster-listing-mm6jmt\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000f87bc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-wdkbpb\\\" in resourcegroup=\\\"cluster-listing-mm6jmt\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"f21283dd9709574e4d2d027adcc67ae3cf590728a17f3f281487b35487b16f76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"a4f110a232d86df240791f6722661fa8cf846b1813ea371fa47312520eb67c98","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"t again to skip cleanup.\n Here's a current progress report:\n Customer should be able to lifecycle and confirm external auth on a cluster (Spec Runtime: 1h29m59.553s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:43\n In [It] (Node Runtime: 1h29m59.553s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:43\n At [By Step] creating HCP cluster (Step Runtime: 34m58.072s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:77\n\n Spec Goroutine\n goroutine 31 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000b4a410}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000b4a410}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0006d6690?}, {{0x824f9c8?, 0xc000d70080?}, 0xc000f48550?}, 0xc00115b5e0, {0xc00022d300, 0xe}, {0xc000671470, 0xd}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func19.2({0x7f4c405d3288, 0xc000932300})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78\n | \n | By(\"creating HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(ctx,\n | \tGinkgoLogr,\n | \t*resourceGroup.Name,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc000932300?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 97\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84 @ 03/10/26 16:42:11.157\n \"ts\"=\"2026-03-10 16:42:11.158116\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.158267\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ea-listgbmwk6-qnr59m\" \"resourceGroup\"=\"ea-list-8kkmkp\"\n \"ts\"=\"2026-03-10 16:42:12.921428\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ea-listgbmwk6-qnr59m\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.881003\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.881160\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.881287\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.883374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-41\"\n \"ts\"=\"2026-03-10 16:42:13.883427\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ea-list-8kkmkp\"\n \"ts\"=\"2026-03-10 16:42:14.674750\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-8kkmkp\" \"deployment\"=\"pooledNonMsiScopedAssignments-3wdubkoimyjyo\"\n \"ts\"=\"2026-03-10 16:42:15.429771\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-41\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:16.199782\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-8kkmkp\" \"deployment\"=\"customer-infra-ea-listgbmwk6-qnr59m\"\n \"ts\"=\"2026-03-10 16:42:16.964086\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-41\" \"deployment\"=\"msi-deployment-41\"\n \"ts\"=\"2026-03-10 16:42:18.268034\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.268266\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.268491\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ea-list-8kkmkp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to lifecycle and confirm external auth on a cluster (Spec Runtime: 1h29m59.553s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:43\n In [It] (Node Runtime: 1h29m59.553s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:43\n At [By Step] creating HCP cluster (Step Runtime: 34m58.072s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:77\n\n Spec Goroutine\n goroutine 31 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x823a1b8, 0xc000b4a410}, 0x781b40d?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x826d180, {0x823a1b8, 0xc000b4a410}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x823a228?, 0xc0006d6690?}, {{0x824f9c8?, 0xc000d70080?}, 0xc000f48550?}, 0xc00115b5e0, {0xc00022d300, 0xe}, {0xc000671470, 0xd}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77fbb54, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func19.2({0x7f4c405d3288, 0xc000932300})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78\n | \n | By(\"creating HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(ctx,\n | \tGinkgoLogr,\n | \t*resourceGroup.Name,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x824faa0?, 0xc000932300?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 97\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84 @ 03/10/26 16:42:11.157\n \"ts\"=\"2026-03-10 16:42:11.158116\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-10 16:42:11.158267\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ea-listgbmwk6-qnr59m\" \"resourceGroup\"=\"ea-list-8kkmkp\"\n \"ts\"=\"2026-03-10 16:42:12.921428\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ea-listgbmwk6-qnr59m\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-10 16:42:13.881003\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.881160\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-399a7bed574b7c4dc2699f197c9d42dfc0866e60ee9f0a810da6aa8ecac44a02.yaml.gz\"\n \"ts\"=\"2026-03-10 16:42:13.881287\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:13.883374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-41\"\n \"ts\"=\"2026-03-10 16:42:13.883427\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ea-list-8kkmkp\"\n \"ts\"=\"2026-03-10 16:42:14.674750\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-8kkmkp\" \"deployment\"=\"pooledNonMsiScopedAssignments-3wdubkoimyjyo\"\n \"ts\"=\"2026-03-10 16:42:15.429771\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-41\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-10 16:42:16.199782\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ea-list-8kkmkp\" \"deployment\"=\"customer-infra-ea-listgbmwk6-qnr59m\"\n \"ts\"=\"2026-03-10 16:42:16.964086\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-41\" \"deployment\"=\"msi-deployment-41\"\n \"ts\"=\"2026-03-10 16:42:18.268034\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-10 16:42:18.268266\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-10 16:42:18.268491\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ea-list-8kkmkp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"fdcbc823dbd2c61be90fb3d641c397c49a3f477121330255b08a5af48588afec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031372817345286144","occurred_at":"2026-03-10T13:59:50Z","failed_at":"e2e","signature_id":"73a56a14ef3be1eef509974253c33880864daabeef59c151c7b537d8d48e2558","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013c63c0\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kdgvhl, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kdgvhl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kdgvhl, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-kdgvhl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009b8460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kdgvhl\",\n },\n \u003c*fmt.wrapError | 0xc000a12240\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-kdgvhl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013c63c0\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kdgvhl, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kdgvhl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kdgvhl, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-kdgvhl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009b8460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kdgvhl\",\n },\n \u003c*fmt.wrapError | 0xc000a12240\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-kdgvhl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":21,"failure_pattern_ids":["4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":21,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031361705358921728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4244,"pr_state":"closed","pr_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","final_merged_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T13:28:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031360693126565888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"9653ebdcfd10d9f3b842a374a4240ebaf05a55a8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T13:25:04Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e4e97e34241985c0e3ec961e9383638d240a599b054de51122057f114dbd5fca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031360693126565888","occurred_at":"2026-03-10T13:25:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4198,"pr_state":"open","pr_sha":"7c96f7415bd5c5912b7e41d2c633adbab42b7503","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T12:47:47Z"},"failed_at":["e2e"],"failed_test_count":16,"occurrences":[{"row_id":"25e022abffcd63ed37831d2e4b168278e1a52c8e56d0650493d5a9c7baa31551","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"0ef4dfc6a022b396b6a332bf12759eb474a7f00c1c5b71cdb2352d52fe53e35f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001644198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwz5kkc\" in resourcegroup=\"ea-list-764klc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b751c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listwz5kkc\\\" in resourcegroup=\\\"ea-list-764klc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b751a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listwz5kkc\\\" in resourcegroup=\\\"ea-list-764klc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b75180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listwz5kkc\\\" in resourcegroup=\\\"ea-list-764klc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001644198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwz5kkc\" in resourcegroup=\"ea-list-764klc\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b751c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listwz5kkc\\\" in resourcegroup=\\\"ea-list-764klc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b751a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listwz5kkc\\\" in resourcegroup=\\\"ea-list-764klc\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b75180\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listwz5kkc\\\" in resourcegroup=\\\"ea-list-764klc\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"32fd006f799a1f065deccba228ede0f7ac5dee60c0a76c80051815d578ec490c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"6187fab253a11d5beebf74cc7036f9c406a89b7e2820ee4d38581cad338fcad9","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f7ee70\u003e: \n failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-q9wg6x\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-q9wg6x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00153ea90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x\",\n },\n \u003c*fmt.wrapErrors | 0xc000f7ee40\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-q9wg6x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00153ea90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f7ee70\u003e: \n failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-q9wg6x\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-q9wg6x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00153ea90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x\",\n },\n \u003c*fmt.wrapErrors | 0xc000f7ee40\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-q9wg6x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00153ea90\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-q9wg6x\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bjwrl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded"}},{"row_id":"56f985d2497f75d6924dfb9af46108f619c929a7b4ba1027dc84ea347ab0a7d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"29901ab460d086a5c2f0a475d63b0b2193008e8642aae5e571a1e89e8639b052","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f1d638\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m7xjc2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0005257c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7xjc2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0005257a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7xjc2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000525780\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7xjc2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000f1d638\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m7xjc2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0005257c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7xjc2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0005257a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7xjc2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000525780\u003e{\n msg: \"failed waiting for hcpCluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-m7xjc2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"5b9844e84acf8d1a8ec466902af480f7466a7d3b9339b74ea3f3370850b281bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"ae771a66faf3c6111399099f39ae4ab8fe2d9c3efd0e7559a06b585ce9357009","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012ef050\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-k2bh7f, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-k2bh7f\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-k2bh7f, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-k2bh7f\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074d310\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-k2bh7f\",\n },\n \u003c*fmt.wrapError | 0xc0009d8780\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-k2bh7f\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012ef050\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-k2bh7f, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-k2bh7f\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-k2bh7f, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-k2bh7f\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074d310\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-k2bh7f\",\n },\n \u003c*fmt.wrapError | 0xc0009d8780\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-k2bh7f\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"5f362e0725eabe766562c2e6ec7ab09bff755295936b985ec57a1e7af1fa5e83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"737b2bda58eb827173711e9b0caf1c022ce97a8f6ab3baf227df57be7252adce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"9a3c45e73b4a8e9a65c3c7175f16e5969bc942374f3ecc38f379af67ff7f14d0","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008984f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-dd8bvh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000990540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-dd8bvh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000990520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-dd8bvh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000990500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-dd8bvh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0008984f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-dd8bvh\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000990540\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-dd8bvh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000990520\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-dd8bvh\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000990500\u003e{\n msg: \"failed waiting for hcpCluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-dd8bvh\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"759957aa822b57cdf10320554d9a777d7a7b8d1c2f22baaf9c455b327b7b1cf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"fd1111c9bd2d5b64ca0897b38b65031b77bcec3e6610b10e86dac8109c43683a","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d82720\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-grbcml\" in resourcegroup=\"cluster-listing-8lw594\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0016413c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-grbcml\\\" in resourcegroup=\\\"cluster-listing-8lw594\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016413a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-grbcml\\\" in resourcegroup=\\\"cluster-listing-8lw594\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001641380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-grbcml\\\" in resourcegroup=\\\"cluster-listing-8lw594\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000d82720\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-grbcml\" in resourcegroup=\"cluster-listing-8lw594\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0016413c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-grbcml\\\" in resourcegroup=\\\"cluster-listing-8lw594\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016413a0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-grbcml\\\" in resourcegroup=\\\"cluster-listing-8lw594\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001641380\u003e{\n msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-grbcml\\\" in resourcegroup=\\\"cluster-listing-8lw594\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"87977553e9fc319117ae3081eb5c4779400f352a9b76b4ffe6d8550ea29dea9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"237e6177bcd2d75153d318552c473a6bf16d21540f2d5927294c0d374962886b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00170e360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g8cnxk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000745c80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-g8cnxk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000745c40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-g8cnxk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000745c20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-g8cnxk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00170e360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g8cnxk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000745c80\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-g8cnxk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000745c40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-g8cnxk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000745c20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-g8cnxk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"9d8018f77be38c1070e6fdc6ca9677034384b360a7621742b083485cefa17063","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"fd6402c822806274f347c802ed182cea34b93d177c1e0ffbba58f5d5f1f5a33d","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00148e450\u003e: \n failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-sxs7cw, error: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-sxs7cw\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-sxs7cw, error: failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sxs7cw\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001371a80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-sxs7cw\",\n },\n \u003c*fmt.wrapError | 0xc0007602e0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sxs7cw\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00148e450\u003e: \n failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-sxs7cw, error: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-sxs7cw\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-one-node, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-sxs7cw, error: failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sxs7cw\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001371a80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-one-node in resource group nodepool-update-nodes-sxs7cw\",\n },\n \u003c*fmt.wrapError | 0xc0007602e0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" for cluster \\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-sxs7cw\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"a1f8ba9d52c2c9a4eae2e09a81a63704fc93d455b431635e5f8360ec1a919a42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"e3e71aae17e56b89d749ccf7b5eb22c0aa10cae23c0c5790cdc467a8905e9337","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016821b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-k8q9jw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007658a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k8q9jw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000765880\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k8q9jw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000765860\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k8q9jw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016821b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-k8q9jw\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0007658a0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k8q9jw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000765880\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k8q9jw\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000765860\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-k8q9jw\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"a9604b2540c819a0a7f993b66f4c21fa01efd9ac294c11a04f7b09b200e5ac71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"e783fa27a9982b1874dc51aef8a83da1542bd0512f14f9c069f65b284c85f269","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e8fc0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lbmfqb, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-lbmfqb\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lbmfqb, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lbmfqb\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00103a230\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lbmfqb\",\n },\n \u003c*fmt.wrapError | 0xc000a12e20\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lbmfqb\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e8fc0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lbmfqb, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-lbmfqb\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lbmfqb, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lbmfqb\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00103a230\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-lbmfqb\",\n },\n \u003c*fmt.wrapError | 0xc000a12e20\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lbmfqb\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-22fc9c, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-22fc9c\" to finish creating: context deadline exceeded"}},{"row_id":"ac7e393bac6f719cca103b19a243817de07f6b0bacf0c8bc15e6888a20f5da7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"c1bce4e08e06988c6bd393157ed502a3bead7af90b6042c008967094308a10b0","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000bf2240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5lmvwj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ea9260\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-5lmvwj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea9240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-5lmvwj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea9220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-5lmvwj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000bf2240\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5lmvwj\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ea9260\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-5lmvwj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea9240\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-5lmvwj\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000ea9220\u003e{\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-5lmvwj\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"ccc23a448a125c2027f1edcf173dbeec19ae83204f50a9444313a87b72e21d4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"ab58faa037cf450849a57b4a5169928e4331f1a8057436c581e143d84035f544","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00122b050\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-kg59h8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-kg59h8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e79440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8\",\n },\n \u003c*fmt.wrapErrors | 0xc00122b020\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-kg59h8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e79440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00122b050\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-kg59h8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-kg59h8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e79440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8\",\n },\n \u003c*fmt.wrapErrors | 0xc00122b020\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-kg59h8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e79440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-kg59h8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bjwrl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded"}},{"row_id":"e115a6445d36253fe2ba73c9b645b53a9b15747a1a9f30da8ed93e2c303abf8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"1349b328683dbf9b10819a4a2b3dd02e28472398260af5d844d8f8166097fafc","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015201f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pb2tlz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000409400\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-pb2tlz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004093e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-pb2tlz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004093c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-pb2tlz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015201f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pb2tlz\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000409400\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-pb2tlz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004093e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-pb2tlz\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0004093c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-pb2tlz\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb0640c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-kkw9j2\" in resourcegroup=\"admin-credential-lifecycle-test-j4vxjb\" to finish deleting: context canceled"}},{"row_id":"e29aea78ad7b4d4fc8cb2bf62c397fde238e48c007dba774015774840a231efd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"7c99ede1b9a43f98bac5674f3546975464c0c0590b69a4fe6a85d3e1432273e4","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001084720\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-6rrcd4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-6rrcd4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001711760\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4\",\n },\n \u003c*fmt.wrapErrors | 0xc0010846f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-6rrcd4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001711760\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001084720\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-6rrcd4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-6rrcd4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001711760\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4\",\n },\n \u003c*fmt.wrapErrors | 0xc0010846f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-6rrcd4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001711760\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-6rrcd4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bjwrl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded"}},{"row_id":"ed85786aaa6473b78c02b98e088371f7e57dd8e84f00bdf61e45fbef16558a92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031351307733831680","occurred_at":"2026-03-10T12:47:47Z","failed_at":"e2e","signature_id":"35c2633b20ee01da0048d3c322bacff00c839f70c82de3af9790fd1872a8ae3e","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000fd5650\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bjwrl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bjwrl4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ee9b60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4\",\n },\n \u003c*fmt.wrapErrors | 0xc000fd5620\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bjwrl4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ee9b60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000fd5650\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bjwrl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bjwrl4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ee9b60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4\",\n },\n \u003c*fmt.wrapErrors | 0xc000fd5620\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bjwrl4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ee9b60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bjwrl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bjwrl4, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031346945389236224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4387,"pr_state":"closed","pr_sha":"5d66b4164b7abb37329e0b6b739ce3ab2d16d8ae","final_merged_sha":"5d66b4164b7abb37329e0b6b739ce3ab2d16d8ae","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T12:30:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4386/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031346153286537216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4386,"pr_state":"closed","pr_sha":"0398ab0c8fead6b99dd5db6edf00beb0269301cb","final_merged_sha":"0398ab0c8fead6b99dd5db6edf00beb0269301cb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T12:27:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031339869279621120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4384,"pr_state":"closed","pr_sha":"f76ac24839d25191cb68347800acec9360017334","final_merged_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T12:02:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031337650425696256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3820,"pr_state":"closed","pr_sha":"7a2f8f6e9d4ad6435c432ea94b519e79a697d572","final_merged_sha":"18929061480414495bf9f44eb8e304a2e2fe45eb","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T11:53:30Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"75d144ea63349422e43c57f8774cd4b6542f744ffc23974ce156b51c6eadc82d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031337650425696256","occurred_at":"2026-03-10T11:53:30Z","failed_at":"e2e","signature_id":"5e8db19d57f5f585ab2c0f84d0fea561d788bf0a53769a22d2ee99b6afdf5cc3","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000858d00\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-c7v86b\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\",\n \"name\": \"73d75999-13f8-466e-b285-0185af102f54\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T12:52:29.933777459Z\",\n \"endTime\": \"2026-03-10T12:54:22.810279491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:730cd492-d01e-007b-1e8c-b0ca06000000\\nTime:2026-03-10T12:53:58.1157027Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c7v86b\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"name\\\": \\\"73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T12:52:29.933777459Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T12:54:22.810279491Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:730cd492-d01e-007b-1e8c-b0ca06000000\\\\nTime:2026-03-10T12:53:58.1157027Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000858ca0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c7v86b\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"name\\\": \\\"73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T12:52:29.933777459Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T12:54:22.810279491Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:730cd492-d01e-007b-1e8c-b0ca06000000\\\\nTime:2026-03-10T12:53:58.1157027Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000858d00\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-c7v86b\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\",\n \"name\": \"73d75999-13f8-466e-b285-0185af102f54\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T12:52:29.933777459Z\",\n \"endTime\": \"2026-03-10T12:54:22.810279491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:730cd492-d01e-007b-1e8c-b0ca06000000\\nTime:2026-03-10T12:53:58.1157027Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c7v86b\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"name\\\": \\\"73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T12:52:29.933777459Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T12:54:22.810279491Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:730cd492-d01e-007b-1e8c-b0ca06000000\\\\nTime:2026-03-10T12:53:58.1157027Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000858ca0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-c7v86b\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"name\\\": \\\"73d75999-13f8-466e-b285-0185af102f54\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T12:52:29.933777459Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T12:54:22.810279491Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:730cd492-d01e-007b-1e8c-b0ca06000000\\\\nTime:2026-03-10T12:53:58.1157027Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ecec28d811d8b3207ea2a77993a0f32c6d481a9e101b16f1bf1d3b6c664fb7ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031337650425696256","occurred_at":"2026-03-10T11:53:30Z","failed_at":"e2e","signature_id":"d1bc0127e3b79e21adc09ee1dce5fe598cd317b94875f40e12af0ab15d01aad6","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000da4b40\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-g7c879\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\",\n \"name\": \"0cf03c0e-11b2-4fad-9131-9411f43c03cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T13:09:21.351421482Z\",\n \"endTime\": \"2026-03-10T13:10:35.544610896Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:f23c12f7-c01e-00f9-558f-b09298000000\\nTime:2026-03-10T13:10:32.3633238Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-g7c879\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"name\\\": \\\"0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T13:09:21.351421482Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T13:10:35.544610896Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f23c12f7-c01e-00f9-558f-b09298000000\\\\nTime:2026-03-10T13:10:32.3633238Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000da4b20\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-g7c879\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"name\\\": \\\"0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T13:09:21.351421482Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T13:10:35.544610896Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f23c12f7-c01e-00f9-558f-b09298000000\\\\nTime:2026-03-10T13:10:32.3633238Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000da4b40\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-g7c879\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\",\n \"name\": \"0cf03c0e-11b2-4fad-9131-9411f43c03cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-10T13:09:21.351421482Z\",\n \"endTime\": \"2026-03-10T13:10:35.544610896Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:f23c12f7-c01e-00f9-558f-b09298000000\\nTime:2026-03-10T13:10:32.3633238Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-g7c879\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"name\\\": \\\"0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T13:09:21.351421482Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T13:10:35.544610896Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f23c12f7-c01e-00f9-558f-b09298000000\\\\nTime:2026-03-10T13:10:32.3633238Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000da4b20\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-g7c879\\\" to finish creating: GET https://rp.j5696256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"name\\\": \\\"0cf03c0e-11b2-4fad-9131-9411f43c03cd\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-10T13:09:21.351421482Z\\\",\\n \\\"endTime\\\": \\\"2026-03-10T13:10:35.544610896Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"GET https://arohcpprowoidcj5696256.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:f23c12f7-c01e-00f9-558f-b09298000000\\\\nTime:2026-03-10T13:10:32.3633238Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4380/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031336521944010752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4380,"pr_state":"closed","pr_sha":"69b2b212c38f98d8751e821a94b6221691685e4c","final_merged_sha":"69b2b212c38f98d8751e821a94b6221691685e4c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T11:48:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031331542676017152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"e680d6c0bf7cbb898516f09eff2fab7a910876d4","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T11:29:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031330853707059200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"e680d6c0bf7cbb898516f09eff2fab7a910876d4","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T11:26:30Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2188bd250f024b8d40ce2b0842f14cd889305fcae81d2bd732f6f27402a09885","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031330853707059200","occurred_at":"2026-03-10T11:26:30Z","failed_at":"e2e","signature_id":"709ec763771e862c0ed82ed4f8a3f185cf70d2bb80b34f9370a5abafbb484cc5","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000988880\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc0008e67e0\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000988880\u003e: \n port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n {\n msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n err: \u003c*errors.errorString | 0xc0008e67e0\u003e{\n s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.11:10250, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031330172510146560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4313,"pr_state":"closed","pr_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","final_merged_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T11:23:43Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031329466172575744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4313,"pr_state":"closed","pr_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","final_merged_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T11:20:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031322628261679104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4384,"pr_state":"closed","pr_sha":"59be66581b75187b36cf49e1458e513d1155b13e","final_merged_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T10:53:49Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a85bb941e6212ccca5e1c59513e5a719bb2d5f32496c3851e412746c0d394f57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031322628261679104","occurred_at":"2026-03-10T10:53:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031321756580450304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4381,"pr_state":"closed","pr_sha":"5ce51466204fb4d0b492344c2f21555a45a836b9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T10:50:01Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6ee6e16f415c9e45ceadbb1ddcf35beea6240ef1057884f840e2c8b61192d6b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031321756580450304","occurred_at":"2026-03-10T10:50:01Z","failed_at":"provision","signature_id":"8ce3a5cb83c2734b2256ab6be017b428155385cd4cacbc4e76eab6ff299daba9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-10T11:41:32.333Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T11:41:32.667Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-10T11:41:33.068Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to fetch pull secret: no credentials found for registry registry.k8s.io in pull secret\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"74e66454e1bf06d12f138b1b9eb0bb1c61e07a63302dc6283650d3a704aecfdd","failure_pattern":"error running Image Mirror Step, failed to fetch pull secret: no credentials found for registry registry.k8s.io in pull secret","search_query":"error running Image Mirror Step, failed to fetch pull secret: no credentials found for registry registry.k8s.io in pull secret"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["74e66454e1bf06d12f138b1b9eb0bb1c61e07a63302dc6283650d3a704aecfdd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031314397271429120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4383,"pr_state":"closed","pr_sha":"eb269d314719909b0563b4f20d851f8af76734f3","final_merged_sha":"9128c3e9a3727d0be6602fb2b928876fd434f14c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T10:21:01Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fb1e66c8179784c13eaa64e447f735bff73fc8ea78c15d51288f2a962229a321","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031314397271429120","occurred_at":"2026-03-10T10:21:01Z","failed_at":"e2e","signature_id":"691bde7eeb092aca0446bac6a4658cf5e143d247ec5d959af030dccc97945b65","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:111]: Timed out after 2400.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f882b8\u003e: \n VerifyHostedControlPlaneVersionUpgradeFrom(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009544e0\u003e{\n msg: \"VerifyHostedControlPlaneVersionUpgradeFrom(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0014725e0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:111]: Timed out after 2400.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f882b8\u003e: \n VerifyHostedControlPlaneVersionUpgradeFrom(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0009544e0\u003e{\n msg: \"VerifyHostedControlPlaneVersionUpgradeFrom(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0014725e0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9e8dea43ec9143e95c656959ef6edb97ae37aebcabdc9abbc1a779473103710","failure_pattern":"VerifyHostedControlPlaneVersionUpgradeFrom(initial=4","search_query":"VerifyHostedControlPlaneVersionUpgradeFrom(initial=4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9e8dea43ec9143e95c656959ef6edb97ae37aebcabdc9abbc1a779473103710"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031314327075557376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"e680d6c0bf7cbb898516f09eff2fab7a910876d4","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T10:20:39Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5a1ff6e7ab56e033daf7468362fa6c86eaba217eb4395926744d5a6f7336b17f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031314327075557376","occurred_at":"2026-03-10T10:20:39Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-10T10:55:50.130Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-10T10:55:53.522Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-10T10:55:55.220Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ea9c323e7e0ed5e03e9ed7dc554923094a25757910fec39e816eea2c3c50a0e9\ntime=2026-03-10T11:08:58.308Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j5557376-svc/providers/Microsoft.Resources/deployments/ea9c323e7e0ed5e03e9ed7dc554923094a25757910fec39e816eea2c3c50a0e9/operationStatuses/08584284667311545815\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/sessiongate-cert-fvino4gogvxv2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCert-sessiongate-cert-prow-j5557376\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentScriptOperationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"A task was canceled.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e816f1e3b47b563efe8a6cd99a62f5abdf0ac3394314f730c55c2f66121c5a89","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentScriptOperationFailed; detail message A task was canceled.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e816f1e3b47b563efe8a6cd99a62f5abdf0ac3394314f730c55c2f66121c5a89"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031313221146644480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4384,"pr_state":"closed","pr_sha":"59be66581b75187b36cf49e1458e513d1155b13e","final_merged_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T10:16:26Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cae6c8e296d875bda60d7a8d346f5a9afd64109a444ce7756a618949cbd4de1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031313221146644480","occurred_at":"2026-03-10T10:16:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031313099729932288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4371,"pr_state":"closed","pr_sha":"11ecb878ec9a3501bb7ba58e755d71f6abf8de21","final_merged_sha":"e0b528c4354bf750dd12c0c682e19c640905138e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T10:15:57Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"03fe0ec846fd51895da4c3e0344bfec6f8eec03c2c6ff5abbcc532952444a518","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031313099729932288","occurred_at":"2026-03-10T10:15:57Z","failed_at":"e2e","signature_id":"75436b9386243811752fc1cc805dea33065f0f1e89b83e5f89a51bef81cd621d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000354780\u003e: \n failed to create NodePool autoscale-np: failed starting nodepool creation \"autoscale-np\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-rqk2f4\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool autoscale-np: failed starting nodepool creation \\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-rqk2f4\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000354740\u003e{\n msg: \"failed starting nodepool creation \\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-rqk2f4\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00039ede0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"94\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"8a8ac7c7-a51b-4490-9f41-b6b26b9ba143\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 11:19:17 GMT\",\n ],\n \"Content-Length\": [\"345\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000354580\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000354780\u003e: \n failed to create NodePool autoscale-np: failed starting nodepool creation \"autoscale-np\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-rqk2f4\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool autoscale-np: failed starting nodepool creation \\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-rqk2f4\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000354740\u003e{\n msg: \"failed starting nodepool creation \\\"autoscale-np\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-rqk2f4\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rqk2f4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-np\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00039ede0\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"94\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"8a8ac7c7-a51b-4490-9f41-b6b26b9ba143\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 11:19:17 GMT\",\n ],\n \"Content-Length\": [\"345\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000354580\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"49e58804605dc3aa1f240763ef104d61d8d503ddfffc2bb295464e8607e6f780","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031313099729932288","occurred_at":"2026-03-10T10:15:57Z","failed_at":"e2e","signature_id":"e90f8134603428e6c4946eb533bbedfee8b4769f4896805bb38024ddd3a28294","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00116a620\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-cnkhnk\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-cnkhnk\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00116a600\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-cnkhnk\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000a93320\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 11:19:25 GMT\",\n ],\n \"Content-Length\": [\"340\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2114\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"0a972fe0-75c5-45bf-a114-5bca2d0dace1\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00116a520\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00116a620\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-cnkhnk\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot create resource while parent resource is provisioning\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-cnkhnk\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00116a600\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-cnkhnk\\\": PUT https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cnkhnk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000a93320\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Tue, 10 Mar 2026 11:19:25 GMT\",\n ],\n \"Content-Length\": [\"340\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2114\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"0a972fe0-75c5-45bf-a114-5bca2d0dace1\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00116a520\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 99, 114, 101, 97, 116, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 119, 104, 105, 108, 101, 32, 112, 97, 114, 101, 110, 116, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 1...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}},{"row_id":"651b790fef6f415b9cfc79ff8febf74fc44f8a8d5d98d45110ec6e38b7b6a83e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031313099729932288","occurred_at":"2026-03-10T10:15:57Z","failed_at":"e2e","signature_id":"4bfe6dfba3750e02d2c8da8181e4b0641cfad496e1beb298f18708df42c1bc21","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:92]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000778540\u003e: \n failed to start credential request: POST https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0007a5170\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"3ac49bae-07be-4982-9e71-34184dd26057\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 11:19:25 GMT\",\n ],\n \"Content-Length\": [\"336\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2038\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000778440\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 110, 111, 100, 101, 112, 111, 111, 108, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 98, 120, 104, 118, 112, 115, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 110, 112, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 104, 99, 112, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 336,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j9932288.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/974ebd46-8ad3-41e3-afef...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:92]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000778540\u003e: \n failed to start credential request: POST https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://rp.j9932288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-bxhvps/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-update-nodes-hcp-cluster\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0007a5170\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"3ac49bae-07be-4982-9e71-34184dd26057\",\n ],\n \"Date\": [\n \"Tue, 10 Mar 2026 11:19:25 GMT\",\n ],\n \"Content-Length\": [\"336\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2038\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000778440\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 110, 111, 100, 101, 112, 111, 111, 108, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 98, 120, 104, 118, 112, 115, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 104, 99, 112, 79, 112, 101, 110, 83, 104, 105, 102, 116, 67, 108, 117, 115, 116, 101, 114, 115, 47, 110, 112, 45, 117, 112, 100, 97, 116, 101, 45, 110, 111, 100, 101, 115, 45, 104, 99, 112, 45, 99, 108, 117, 115, 116, 101, 114, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 336,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j9932288.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/974ebd46-8ad3-41e3-afef...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0f2f71076ecda30b6ae0d13512c44eed7037c704b65b8be7109f64fb982271ac","491f7ba60c419a0faa324d3b03672a02667d75e6ba31bc1048f23be270bd76a2"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4329/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031312183714910208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4329,"pr_state":"closed","pr_sha":"0289e5cba1076ce737ad437633b268498778a383","final_merged_sha":"0289e5cba1076ce737ad437633b268498778a383","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T10:12:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4380/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031307494369792000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4380,"pr_state":"closed","pr_sha":"88b6e6bb6aae4aa3e20fb31c5af2456807509fd9","final_merged_sha":"69b2b212c38f98d8751e821a94b6221691685e4c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T09:53:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1d3c21537f4c1c69b97a4043973b8a026b2c75a137cbfaaa8a0efce3cdb1de2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4380/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031307494369792000","occurred_at":"2026-03-10T09:53:37Z","failed_at":"provision","signature_id":"55df8ef767832357b4eed91c76c181e04e89cd243d85f72d5f7ac8ee4f9cef71","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","test_suite":"step graph","failure_text":"time=2026-03-10T10:40:57.288Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter description=\"Step aro-hcp-exporter\\n Kind: Helm\\n\"\ntime=2026-03-10T10:40:57.770Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter values=\"map[cacheTTL:60m collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:[service-tag-usage] image:map[digest:sha256:2a64b8900486cd886d2c44023788f95891a3012ad790f85272432a30685e2a34 registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] listenAddress::8080 service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:9df125b8-466a-45c8-a7b7-9eeb6183562f name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-10T10:40:57.770Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T10:40:58.100Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter namespace=aro-hcp-exporter\ntime=2026-03-10T10:40:58.340Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter dryRun=false\ntime=2026-03-10T10:40:58.340Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T10:40:58.456Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T10:46:01.741Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-10T10:46:01.741Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T10:46:01.741Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T10:46:04.076Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter release=aro-hcp-exporter namespace=aro-hcp-exporter status=failed description=\"Release \\\"aro-hcp-exporter\\\" failed: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[cacheTTL:60m collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:[service-tag-usage] image:map[digest:sha256:2a64b8900486cd886d2c44023788f95891a3012ad790f85272432a30685e2a34 registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] listenAddress::8080 service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:9df125b8-466a-45c8-a7b7-9eeb6183562f name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-10T10:46:04.078Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yRQWsqMRSF9/kVl1k5MCOj7z15THFRaFel3bQrSxeZ5KDRmdyQXLWCP75MpdUWhNrtl3tOkvupFkIRidfRINGUrBYtumkx4GYJI3fO2zpJdH5e0AE96A6fCBv4d5KCNh84f1ZERNkj4sYZ3LN3wjErKNORy4UJJV4DR8EZ9iV9bQyvvfwyfWnsBqHlXYcLLlQvV2q1bhA9BOm2X0hSe9ouEEHiOiTRXaAGsgU8DawW9HiQjavxpKz+lKPqaVTVf6v63/9ZltNwSGdnJnU1mmV5rva0ZOdp5bydOu8RTzSyp6O9U23ffak9hcj96fGhBQW2h+GflRTU8rygDinpeV/J0SJSszv5vEUybwEAAP//A4U572wCAAA=\"\ntime=2026-03-10T10:46:04.080Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T10:46:04.080Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25","failure_pattern":"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031297984615944192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4244,"pr_state":"closed","pr_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","final_merged_sha":"e7f1e378ae58602463a0b616f3ea3534bbfd785f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T09:15:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031294345923006464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T09:01:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4379/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031292244664782848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4379,"pr_state":"closed","pr_sha":"18b2accec3bcf392f999715dfeab7a58405a00e1","final_merged_sha":"18b2accec3bcf392f999715dfeab7a58405a00e1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T08:53:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031291493750149120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4366,"pr_state":"closed","pr_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","final_merged_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T08:50:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031290805355810816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4382,"pr_state":"closed","pr_sha":"0f5cde21dc9346ce0a0483ed00227a5af90a2f5c","final_merged_sha":"0f5cde21dc9346ce0a0483ed00227a5af90a2f5c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T08:47:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031283150205751296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4381,"pr_state":"closed","pr_sha":"a77910e938b0db6ef47a549edb79c51275792852","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T08:16:57Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"91e638a85b3e95c1427edd1e0f1d0531ba92a1efbf1d6774e26eb3b611e72944","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031283150205751296","occurred_at":"2026-03-10T08:16:57Z","failed_at":"provision","signature_id":"79a3108729f620eb0ff2a0cbe6c80039eb4fc8c409ca18f17f47cb438751ae7d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-10T09:01:37.170Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T09:01:37.468Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-10T09:01:38.017Z level=INFO msg=\"Failed to fetch pull secret, will try anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image error=\"no credentials found for registry registry.build02.ci.openshift.org in pull secret\"\ntime=2026-03-10T09:01:38.017Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-10T09:01:39.590Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image source=registry.build02.ci.openshift.org/ci-op-9bchqrkl/pipeline@sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f target=arohcpsvcdev.azurecr.io/ci-op-9bchqrkl/pipeline:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\ntime=2026-03-10T09:01:39.697Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:02:09.773Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:02:39.861Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:03:39.862Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T09:03:40.158Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-10T09:03:40.768Z level=INFO msg=\"Failed to fetch pull secret, will try anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image error=\"no credentials found for registry registry.build02.ci.openshift.org in pull secret\"\ntime=2026-03-10T09:03:40.768Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-10T09:03:42.242Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image source=registry.build02.ci.openshift.org/ci-op-9bchqrkl/pipeline@sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f target=arohcpsvcdev.azurecr.io/ci-op-9bchqrkl/pipeline:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\ntime=2026-03-10T09:03:42.293Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:04:12.489Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:04:42.539Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:05:42.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T09:05:42.830Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-10T09:05:43.385Z level=INFO msg=\"Failed to fetch pull secret, will try anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image error=\"no credentials found for registry registry.build02.ci.openshift.org in pull secret\"\ntime=2026-03-10T09:05:43.385Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-10T09:05:45.031Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image source=registry.build02.ci.openshift.org/ci-op-9bchqrkl/pipeline@sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f target=arohcpsvcdev.azurecr.io/ci-op-9bchqrkl/pipeline:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\ntime=2026-03-10T09:05:45.053Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:06:15.089Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:06:45.142Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:07:45.143Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T09:07:45.513Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-10T09:07:46.191Z level=INFO msg=\"Failed to fetch pull secret, will try anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image error=\"no credentials found for registry registry.build02.ci.openshift.org in pull secret\"\ntime=2026-03-10T09:07:46.191Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-10T09:07:47.689Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image source=registry.build02.ci.openshift.org/ci-op-9bchqrkl/pipeline@sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f target=arohcpsvcdev.azurecr.io/ci-op-9bchqrkl/pipeline:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\ntime=2026-03-10T09:07:47.777Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:08:17.828Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:08:47.892Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:09:47.894Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-10T09:09:48.182Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-10T09:09:48.931Z level=INFO msg=\"Failed to fetch pull secret, will try anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image error=\"no credentials found for registry registry.build02.ci.openshift.org in pull secret\"\ntime=2026-03-10T09:09:48.931Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-10T09:09:50.464Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image source=registry.build02.ci.openshift.org/ci-op-9bchqrkl/pipeline@sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f target=arohcpsvcdev.azurecr.io/ci-op-9bchqrkl/pipeline:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\ntime=2026-03-10T09:09:50.531Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:10:20.600Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:10:50.718Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"\ntime=2026-03-10T09:11:50.723Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c8c2ff3d8c303914240c5726c406b21f7ab297ba660a555c1bcaa2a0ef7b47a6","failure_pattern":"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \"FetchReference\" on source: GET \"\u003curl\u003e\": response status code 401: unauthorized: authentication required: [map[Action:pull Class","search_query":"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"FetchReference\\\" on source: GET \\\"https://registry.build02.ci.openshift.org/v2/ci-op-9bchqrkl/pipeline/manifests/sha256:c982aa0bfa01bbbecab3f1f77e10f1aae554ededa3d0d30fada49049423a091f\\\": response status code 401: unauthorized: authentication required: [map[Action:pull Class: Name:ci-op-9bchqrkl/pipeline Type:repository]]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c8c2ff3d8c303914240c5726c406b21f7ab297ba660a555c1bcaa2a0ef7b47a6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4380/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031281249921798144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4380,"pr_state":"closed","pr_sha":"88b6e6bb6aae4aa3e20fb31c5af2456807509fd9","final_merged_sha":"69b2b212c38f98d8751e821a94b6221691685e4c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-10T08:09:15Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f8a09ca2c91e4656a9420afffe4097457c3f86e73882e9fb243ac6a816dc059a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4380/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031281249921798144","occurred_at":"2026-03-10T08:09:15Z","failed_at":"provision","signature_id":"55df8ef767832357b4eed91c76c181e04e89cd243d85f72d5f7ac8ee4f9cef71","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","test_suite":"step graph","failure_text":"time=2026-03-10T09:14:23.191Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter description=\"Step aro-hcp-exporter\\n Kind: Helm\\n\"\ntime=2026-03-10T09:14:23.917Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter values=\"map[cacheTTL:60m collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:[service-tag-usage] image:map[digest:sha256:2a64b8900486cd886d2c44023788f95891a3012ad790f85272432a30685e2a34 registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] listenAddress::8080 service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:e577ae7c-f860-47b0-91ba-c263f6adcacc name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-10T09:14:23.917Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T09:14:24.102Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter namespace=aro-hcp-exporter\ntime=2026-03-10T09:14:24.322Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter dryRun=false\ntime=2026-03-10T09:14:24.322Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T09:14:24.429Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T09:19:27.307Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-10T09:19:27.308Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T09:19:27.308Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T09:19:29.663Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter release=aro-hcp-exporter namespace=aro-hcp-exporter status=failed description=\"Release \\\"aro-hcp-exporter\\\" failed: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[cacheTTL:60m collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:[service-tag-usage] image:map[digest:sha256:2a64b8900486cd886d2c44023788f95891a3012ad790f85272432a30685e2a34 registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] listenAddress::8080 service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:e577ae7c-f860-47b0-91ba-c263f6adcacc name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-10T09:19:29.664Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yRQU/yQBCG7/0Vk55o0pLSj3yGGg4mejJ60RPGw3b3DSy0O5vdASThx5tKFGIkEq/Pvu/s7jxJC6GAyOugEWlKRokS1bQYcLOElnvrTB0lWDfP6YAeVYcvhA3cB4le6U+cvSREROktfMu7Dk7SnFIVuFhoX+DNcxCEn9mh+YSwsRoP7KzwmeSv7Rutef3nuy+uJa/XyWrdIDgI4l2/kJjsabtAAIntEEV1nhrIFnA0MErQ40FaldX/ovxXjMrnclKPxnU1nqUZDYd0NjOpq6tZmmXJnpZsHa2sM1PrHMKJRnZ0tHeq7buvZE8+cH96fGhOns0hfNmQnFqe59QhRjXvR3IwCNTsTj5vEPV7AAAA//+dHcuZbAIAAA==\"\ntime=2026-03-10T09:19:29.668Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-10T09:19:29.669Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25","failure_pattern":"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031279910235934720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4313,"pr_state":"closed","pr_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","final_merged_sha":"40bc11d7af0e82eb8a36a7b77d76c911d23c2191","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T08:04:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031262327323496448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4363,"pr_state":"closed","pr_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","final_merged_sha":"523a271a3a332b29ad83ffe992284dea47e7a593","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T06:54:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031258484850298880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4366,"pr_state":"closed","pr_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","final_merged_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T06:38:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4379/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031254991418691584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4379,"pr_state":"closed","pr_sha":"18b2accec3bcf392f999715dfeab7a58405a00e1","final_merged_sha":"18b2accec3bcf392f999715dfeab7a58405a00e1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T06:25:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031227661962973184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4378,"pr_state":"closed","pr_sha":"86cf6cad462793f76f68b676a94a68faefbc1672","final_merged_sha":"86cf6cad462793f76f68b676a94a68faefbc1672","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T04:36:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031213544657129472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4272,"pr_state":"open","pr_sha":"66a51589dab1c535676aba03930d89302e1c5b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T03:40:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4366/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031213432274948096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4366,"pr_state":"closed","pr_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","final_merged_sha":"c7c9d81a876290f11802e68dbceb40c40dfd998e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T03:39:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4372/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031206511748845568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4372,"pr_state":"closed","pr_sha":"e680d6c0bf7cbb898516f09eff2fab7a910876d4","final_merged_sha":"506b932c6f494bfd84965a40bd51ed4a28647d20","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-10T03:12:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031195200944082944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4378,"pr_state":"closed","pr_sha":"86cf6cad462793f76f68b676a94a68faefbc1672","final_merged_sha":"86cf6cad462793f76f68b676a94a68faefbc1672","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T02:27:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4374/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031170984882475008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4374,"pr_state":"closed","pr_sha":"d1097e4d55cb9d3f37d85bd1401446e7ff42fefd","final_merged_sha":"d1097e4d55cb9d3f37d85bd1401446e7ff42fefd","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T00:51:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4377/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2031167595394109440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4377,"pr_state":"closed","pr_sha":"ee089147f8b859bcf62d9c5b485d911cafe1412c","final_merged_sha":"ee089147f8b859bcf62d9c5b485d911cafe1412c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-10T00:37:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}