{"meta":{"date":"2026-03-18","timezone":"UTC","generated_at":"2026-10-31T23:15:13Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":76,"failed_runs":49,"runs_with_occurrences":49,"runs_with_failure_pattern_matches":38,"runs_with_unmatched_occurrences":11,"runs_without_occurrence_rows":27,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034416501842251776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4511,"pr_state":"closed","pr_sha":"39d25fe41a7f321f042998103d74578dce0f5bc4","final_merged_sha":"39d25fe41a7f321f042998103d74578dce0f5bc4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T23:47:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034413767453839360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4396,"pr_state":"closed","pr_sha":"732c1753890fbef90e84268733f7e4d18f32e691","final_merged_sha":"732c1753890fbef90e84268733f7e4d18f32e691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T23:36:54Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"292875d944f85d12f06b0960002ba76c4a9fba1009a2e28175f908733c7ce17d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034413767453839360","occurred_at":"2026-03-18T23:36:54Z","failed_at":"e2e","signature_id":"28f62afe1b504e90cea5fe4021f66bb2340457ea7a3f9c97c959d36cfd23a4cf","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000800ae0\u003e: \n    port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n    {\n        msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n        err: \u003c*errors.errorString | 0xc000a68cd0\u003e{\n            s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000800ae0\u003e: \n    port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\n    {\n        msg: \"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n        err: \u003c*errors.errorString | 0xc000a68cd0\u003e{\n            s: \"error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a","failure_pattern":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.8:10250, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f27efe24b36721c3037189b0270b3c037f3dcd8e282db885731d7395d97e1e2a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034398436282863616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4493,"pr_state":"closed","pr_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","final_merged_sha":"8a3e2673bc025c3e97823e6867d3b9ede213d29b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T22:35:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034395794701815808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T22:25:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034387593050722304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4484,"pr_state":"closed","pr_sha":"9cfdfc7fd42f86ed416f4ce7fc4f32ed70846bd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T21:52:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034387547781599232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"f27ad3088a570cce40c49a9520677598322a54d9","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T21:52:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d77c2842f50eb85b42d45db56c1a2aff945d0e465c03cd5fb1bfe4ad70a6b27b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034387547781599232","occurred_at":"2026-03-18T21:52:43Z","failed_at":"e2e","signature_id":"9093b17d5c973c6ce25b91108a39783cce2bed752eaa0c1efbad4072f56e2687","test_name":"Customer should upgrade a nodepool from 4.20 to 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:178]: Expected\n    \u003cstring\u003e: 4.20.16\nto equal\n    \u003cstring\u003e: 4.21.5\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:178]: Expected\n    \u003cstring\u003e: 4.20.16\nto equal\n    \u003cstring\u003e: 4.21.5","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ef0331f04181565e0d0a1368dd8c037a9da21a69d62675a569661e49234fd17","failure_pattern":"\u003cstring\u003e: 4.21.5","search_query":"failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3ef0331f04181565e0d0a1368dd8c037a9da21a69d62675a569661e49234fd17"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034385514982805504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"a5c42f1b529a340ee88dafbab46287500af591d1","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T21:44:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4511/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034382639141490688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4511,"pr_state":"closed","pr_sha":"39d25fe41a7f321f042998103d74578dce0f5bc4","final_merged_sha":"39d25fe41a7f321f042998103d74578dce0f5bc4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T21:33:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034382507851386880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4512,"pr_state":"closed","pr_sha":"ca7bbc980a35e3fc6dc73742e2b24e1d36f1178a","final_merged_sha":"ca7bbc980a35e3fc6dc73742e2b24e1d36f1178a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T21:32:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4396/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034379943080628224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4396,"pr_state":"closed","pr_sha":"732c1753890fbef90e84268733f7e4d18f32e691","final_merged_sha":"732c1753890fbef90e84268733f7e4d18f32e691","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T21:22:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034364782110314496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"c84eed08362bdc95a6bee238819cb7de596fcb69","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T20:22:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034363879517065216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T20:18:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034363593817853952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T20:17:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034356765943926784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4519,"pr_state":"closed","pr_sha":"3346aec94a705a767ceaec1df24dfecc2108be77","final_merged_sha":"3346aec94a705a767ceaec1df24dfecc2108be77","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T19:50:24Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c3386de6d80905bee0804bf34ec357244bc0a39290dfd13c8c9610ec2e1be27e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034356765943926784","occurred_at":"2026-03-18T19:50:24Z","failed_at":"e2e","signature_id":"ac6af0352f377fda4e14c7a08f5770e246d69685d523366e70ff5ba62cfca0d1","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000da0f80\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000da0f80\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034354948560392192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4391,"pr_state":"open","pr_sha":"dacf87647418e205a2b119537d656f914a585006","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T19:43:10Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5cdc9dfbc1575f477b51be03b152e65b67a34c7391f651925439104d76395b0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034354948560392192","occurred_at":"2026-03-18T19:43:10Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034348901456154624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4391,"pr_state":"open","pr_sha":"dacf87647418e205a2b119537d656f914a585006","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T19:19:09Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9e75773967973d1764f69c7fc8565fc65f17f33a3c2b307eae796ad056010a9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4391/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034348901456154624","occurred_at":"2026-03-18T19:19:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034338735830601728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"ea2b721339c793e40b62ef6bf53f849b90146e55","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T18:38:45Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4886e046b9f04e6a1650a716499610ad42a55fa55c20ebc7bc80008c1a075217","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034338735830601728","occurred_at":"2026-03-18T18:38:45Z","failed_at":"e2e","signature_id":"b5843a8ed73d1bd562cc7995d8dae1ecd02b48839b35392dfa2fc42a7add4e30","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000a077d0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000a077d0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"b0ec07c6922301aaa753f4c92c91b17b72b4241b1023b6016b0c83f41cab4f5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034338735830601728","occurred_at":"2026-03-18T18:38:45Z","failed_at":"e2e","signature_id":"9093b17d5c973c6ce25b91108a39783cce2bed752eaa0c1efbad4072f56e2687","test_name":"Customer should upgrade a nodepool from 4.20 to 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:180]: Expected\n    \u003cstring\u003e: 4.20.16\nto equal\n    \u003cstring\u003e: 4.21.5\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:180]: Expected\n    \u003cstring\u003e: 4.20.16\nto equal\n    \u003cstring\u003e: 4.21.5","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ef0331f04181565e0d0a1368dd8c037a9da21a69d62675a569661e49234fd17","failure_pattern":"\u003cstring\u003e: 4.21.5","search_query":"failure"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["3ef0331f04181565e0d0a1368dd8c037a9da21a69d62675a569661e49234fd17","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034338032965914624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4517,"pr_state":"closed","pr_sha":"a5c42f1b529a340ee88dafbab46287500af591d1","final_merged_sha":"98f9b5f64bc2a1090552d4cecf1ed332bfca4775","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T18:35:55Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0c8eaa395c87c8e43b6007f495bc3ea2be67703d3f94b0d58b5e868d8e41dd1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034338032965914624","occurred_at":"2026-03-18T18:35:55Z","failed_at":"e2e","signature_id":"3dbb93b88d553134012e1f0bc2ddffcfad1d63ecdfd97cf42466c7cd9bfcac22","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000511410\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000511410\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"53c8d64750426ddb0958d97166032ff7b6bfc13700dfc63d85466a8e8abf2d32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034338032965914624","occurred_at":"2026-03-18T18:35:55Z","failed_at":"e2e","signature_id":"317206074d6a20f2f77868c4caabc81ea6d0f7bb570e63d2b5e01b021bc43ca5","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:107]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c48e70\u003e: \n    VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0012fb220\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n                err: \u003c*errors.errorString | 0xc000a0b210\u003e{\n                    s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:107]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c48e70\u003e: \n    VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0012fb220\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n                err: \u003c*errors.errorString | 0xc000a0b210\u003e{\n                    s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","failure_pattern":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then","search_query":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034337208256696320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4133,"pr_state":"closed","pr_sha":"6b929a55593057d1fd605ea8ba3b55fd5521d131","final_merged_sha":"4f10ef55a408ea39a4f644c08d4ab2d01cf39c2a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T18:32:41Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"69fae50c52f7adb9ec1ee15d5a7f62163534c130935a2906300f92625e2ddb9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034337208256696320","occurred_at":"2026-03-18T18:32:41Z","failed_at":"e2e","signature_id":"9d3df3d4dbc485c073934e2ff39f67d43f6df32ae1240624a8e676f7ce33fb4b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to 500 replicas","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:134]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962c60\u003e: \n    failed to create NodePool autoscale-az: failed starting nodepool creation \"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9ntvp4\": PUT https://rp.j6696320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9ntvp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool autoscale-az: failed starting nodepool creation \\\"autoscale-az\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9ntvp4\\\": PUT https://rp.j6696320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9ntvp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000962c40\u003e{\n            msg: \"failed starting nodepool creation \\\"autoscale-az\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9ntvp4\\\": PUT https://rp.j6696320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9ntvp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000b967b0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"0db94d04-dd8e-47c4-a152-06aa92aab279\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 19:48:49 GMT\",\n                        ],\n                        \"Content-Length\": [\"283\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"1179\"],\n                        \"Server\": [\"istio-envoy\"],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000962b80\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 112, 108, 105, 99, 97, 115, 43, 65, 117, 116, 111, 115, 99, 97, 108, 105, 110, 103, 46, 77, 97, 120, 58, 32, 84, 104, 101, 32, 116, 111, 116, 97, 108, 32, 110, 117, 109, 98, 101, 114, 32, 111, 102, 32, 99, 111, 109, 112, 117, 116, 101, 32, 110, 111, 100, 101, 115, 32, 102, 111, 114, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 99, 108, 117, 115, 116, 101, 114, 32, 39, 55, 48, 48, 39, 32, 101, 120, 99, 101, 101, 100, 115, 32, 116, 104, 101, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:134]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962c60\u003e: \n    failed to create NodePool autoscale-az: failed starting nodepool creation \"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9ntvp4\": PUT https://rp.j6696320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9ntvp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool autoscale-az: failed starting nodepool creation \\\"autoscale-az\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9ntvp4\\\": PUT https://rp.j6696320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9ntvp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000962c40\u003e{\n            msg: \"failed starting nodepool creation \\\"autoscale-az\\\" for cluster \\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-9ntvp4\\\": PUT https://rp.j6696320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9ntvp4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000b967b0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"0db94d04-dd8e-47c4-a152-06aa92aab279\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 19:48:49 GMT\",\n                        ],\n                        \"Content-Length\": [\"283\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"1179\"],\n                        \"Server\": [\"istio-envoy\"],\n                    },\n                    Body: \u003c*exported.nopClosingBytesReader | 0xc000962b80\u003e{\n                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 112, 108, 105, 99, 97, 115, 43, 65, 117, 116, 111, 115, 99, 97, 108, 105, 110, 103, 46, 77, 97, 120, 58, 32, 84, 104, 101, 32, 116, 111, 116, 97, 108, 32, 110, 117, 109, 98, 101, 114, 32, 111, 102, 32, 99, 111, 109, 112, 117, 116, 101, 32, 110, 111, 100, 101, 115, 32, 102, 111, 114, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 99, 108, 117, 115, 116, 101, 114, 32, 39, 55, 48, 48, 39, 32, 101, 120, 99, 101, 101, 100, 115, 32, 116, 104, 101, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c5332bfefe86cbb7c6fc9ad74dd945b0289317f49b706bc63f98022c726f913","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Replicas+Autoscaling.Max: The total number of compute nodes for a single cluster '700' exceeds the maximum allowed '500'. Reduce the total compute nodes requested to be within the maximum allowed.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cdcf71401dcce4bd7a4655aa052d6ce1b9792512eb2fe1eb531199c7d1f74494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034337208256696320","occurred_at":"2026-03-18T18:32:41Z","failed_at":"e2e","signature_id":"cca112c48237ff28e13d0e730f4b4ff1c8a653f2c6f63433d435906f1674c9b3","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001620070\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001620070\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","8c5332bfefe86cbb7c6fc9ad74dd945b0289317f49b706bc63f98022c726f913"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034335130373328896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4484,"pr_state":"closed","pr_sha":"9cfdfc7fd42f86ed416f4ce7fc4f32ed70846bd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T18:24:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0e777d47c62088eee5801e6d33ac397c0775522991e45743cba09b76fdbb78d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034335130373328896","occurred_at":"2026-03-18T18:24:25Z","failed_at":"e2e","signature_id":"fa2d1021ae5298ebe68db7e61b5a056f032f8a33b7ac1f15b4f9fed6c282d75a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000af7a10\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000af7a10\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034332131752153088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T18:12:30Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"135a2aedff6eab4147d35b7a3f1ea7a62968f53037db8bd20e95784e8149b1ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034332131752153088","occurred_at":"2026-03-18T18:12:30Z","failed_at":"e2e","signature_id":"aba1ef59c8fc8f002f8c5808ae07706857a79a9773e5d6434c9e874390889785","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009fda40\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-dd4h2h\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-dd4h2h\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009e77a0\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009fda40\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-dd4h2h\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-dd4h2h\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009e77a0\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-dd4h2h\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h, error: context deadline exceeded"}},{"row_id":"345f960041afea677e666e28b7e1e2c942818030dc0f06290906f11f32b41761","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034332131752153088","occurred_at":"2026-03-18T18:12:30Z","failed_at":"e2e","signature_id":"df023ed14f2479d9e7967d3f5605aa2db14f4870a2d8631da28cf7994c912812","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b8e160\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-pnpwp.apps.aro.x7y4h7y8r4t2i2c.bnid.j2153088.hcp.osadev.cloud\": dial tcp 4.236.24.249:443: connect: connection timed out\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-pnpwp.apps.aro.x7y4h7y8r4t2i2c.bnid.j2153088.hcp.osadev.cloud\\\": dial tcp 4.236.24.249:443: connect: connection timed out\",\n        err: \u003c*url.Error | 0xc000daa0f0\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-pnpwp.apps.aro.x7y4h7y8r4t2i2c.bnid.j2153088.hcp.osadev.cloud\",\n            Err: \u003c*net.OpError | 0xc0009ea140\u003e{\n                Op: \"dial\",\n                Net: \"tcp\",\n                Source: nil,\n                Addr: \u003c*net.TCPAddr | 0xc000daa300\u003e{\n                    IP: [4, 236, 24, 249],\n                    Port: 443,\n                    Zone: \"\",\n                },\n                Err: \u003c*os.SyscallError | 0xc000b8e120\u003e{\n                    Syscall: \"connect\",\n                    Err: \u003csyscall.Errno\u003e0x6e,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b8e160\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-pnpwp.apps.aro.x7y4h7y8r4t2i2c.bnid.j2153088.hcp.osadev.cloud\": dial tcp 4.236.24.249:443: connect: connection timed out\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-pnpwp.apps.aro.x7y4h7y8r4t2i2c.bnid.j2153088.hcp.osadev.cloud\\\": dial tcp 4.236.24.249:443: connect: connection timed out\",\n        err: \u003c*url.Error | 0xc000daa0f0\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-pnpwp.apps.aro.x7y4h7y8r4t2i2c.bnid.j2153088.hcp.osadev.cloud\",\n            Err: \u003c*net.OpError | 0xc0009ea140\u003e{\n                Op: \"dial\",\n                Net: \"tcp\",\n                Source: nil,\n                Addr: \u003c*net.TCPAddr | 0xc000daa300\u003e{\n                    IP: [4, 236, 24, 249],\n                    Port: 443,\n                    Zone: \"\",\n                },\n                Err: \u003c*os.SyscallError | 0xc000b8e120\u003e{\n                    Syscall: \"connect\",\n                    Err: \u003csyscall.Errno\u003e0x6e,\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: connect: connection timed out","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\": dial tcp 51.57.113.19:443: connect: connection timed out"}},{"row_id":"ac7d8f4125a0a26baf2a266bd6ee9271cf3c39f1eea3ec2c799c48391a781557","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034332131752153088","occurred_at":"2026-03-18T18:12:30Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"c7f5fe0c4325aafbc29f873bdd39e109eb6a92d3670e125f84874fbdaeceba0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034332131752153088","occurred_at":"2026-03-18T18:12:30Z","failed_at":"e2e","signature_id":"f58411d43bffa62152516abc1f5d57520a805302d91f55df4b9edfb54b5178d2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00072aa00\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00072aa00\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4157,"pr_state":"open","pr_sha":"e5f8a5df688d4e970aeef2aab9bf09406a8ee464","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T18:09:09Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"002d6ade2fc4722eb1b97fc3e492ea4a2590cef0ebece663de07814454f311f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"4b450ef8de97839dca3b0dcfa55756f1efd0155300dba2795b820233995327b2","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001001b90\u003e: \n    failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-gtlnpg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-gtlnpg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-gtlnpg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-gtlnpg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000bf2720\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-gtlnpg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001001b90\u003e: \n    failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-gtlnpg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-gtlnpg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-gtlnpg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-gtlnpg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000bf2720\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-gtlnpg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"039ad7f6a1ba6d4c194ffd3c28cb1ff42b0a33e88e70ee7b70f9b18cafc0cca8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"dc3feef26d9818bd15c4a9d575eb9ae98784b6db625ba881247058f5b9efa600","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001465590\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-hlgz5x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4scpw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011320f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001465590\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-hlgz5x\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j4scpw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011320f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"07dd1fc70aa3c2af194267bc631212dfefadabca5c72f9f9d904b00ab4aa2b5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"663c39d603b2bfcbdac7cfb1fb634abaaa47911e4d6777ca0d1898c59f331141","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f301f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nkn7p\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0007e6d80\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7nkn7p\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0007e6d60\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7nkn7p\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0007e6d40\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7nkn7p\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f301f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7nkn7p\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0007e6d80\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7nkn7p\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0007e6d60\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7nkn7p\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0007e6d40\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-7nkn7p\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"09c966b165e348981c3f571e5d37b58222067cd9eae3a4fd2570886672707711","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"fd250b09e993ff8ce4a3c4b60e2fbf2c65ea4fca2f20fba6a1fec4c6320bbfa6","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000158c30\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-ddvp7b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-ddvp7b, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-ddvp7b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-ddvp7b, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000581720\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-ddvp7b\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000158c30\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-ddvp7b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-ddvp7b, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-ddvp7b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-ddvp7b, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000581720\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-ddvp7b\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"1d8a49b4f47a798aab03b3fe1e5ae8d099e536efd7aab01c424c9fd8c60e7b7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"03634a72e0c4d714835aea9c35fe4a26a38e400ff4a1fc2fd75e682ec43d2c23","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f9e330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-lpr96q\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001616f40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-lpr96q\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001616f20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-lpr96q\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc001616f00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-lpr96q\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f9e330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-lpr96q\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001616f40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-lpr96q\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001616f20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-lpr96q\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc001616f00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-lpr96q\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"29505797d8ab98aa1ecc37c2019bbbbda56334b6d89fdfd27e6567bf3bb730cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"0884cbd510371328a8d8330f7bd2b3525ddc997b3fe3cdf9ac1ade72e7d1264e","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010ca210\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-w6bnfq\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00094dd40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-w6bnfq\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00094dd20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-w6bnfq\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00094dd00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-w6bnfq\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010ca210\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-w6bnfq\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00094dd40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-w6bnfq\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00094dd20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-w6bnfq\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00094dd00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-w6bnfq\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"2d4bb61c0e2fea5739a14a54ea8cc83acc784470d1919f21f0cda39c3053e850","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"8276cb09992231e3ad6884d20cf72bb6f80bc35e801690646ed18c4c102debcc","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00020b1b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-4ggzfv\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008b7920\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4ggzfv\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008b7900\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4ggzfv\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0008b78e0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4ggzfv\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00020b1b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-4ggzfv\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008b7920\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4ggzfv\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008b7900\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4ggzfv\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0008b78e0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4ggzfv\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"2f7aea77617a2de43990c8df4065b62ce8b901614202f996e8d8db9c781841da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"9b7a45afa2e3c8e13e27b0466704abd156fe458e4bc802c8a0dacafd50d8e88c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:115]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ba480\u003e: \n    failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vsqvd2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-vsqvd2, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-vsqvd2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-vsqvd2, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e15260\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-vsqvd2\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:115]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ba480\u003e: \n    failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vsqvd2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-vsqvd2, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-vsqvd2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-vsqvd2, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000e15260\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-vsqvd2\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"2f931ab8175a18fefd70c7632f04d4c86a9ac9e04bc04d64c3f410a1a51b21cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"9cc3acd71c5342d66f0d578737cf1d4d578163efcdf6193595814ec707e80425","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00124a600\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-8tn56p\" in resourcegroup=\"rg-zstream-upgrade-4-20-fhqqrg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-8tn56p in resource group rg-zstream-upgrade-4-20-fhqqrg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-8tn56p\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-fhqqrg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-8tn56p in resource group rg-zstream-upgrade-4-20-fhqqrg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001357d00\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-8tn56p in resource group rg-zstream-upgrade-4-20-fhqqrg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00124a600\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-8tn56p\" in resourcegroup=\"rg-zstream-upgrade-4-20-fhqqrg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-8tn56p in resource group rg-zstream-upgrade-4-20-fhqqrg, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-8tn56p\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-fhqqrg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-8tn56p in resource group rg-zstream-upgrade-4-20-fhqqrg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001357d00\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-8tn56p in resource group rg-zstream-upgrade-4-20-fhqqrg\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"31c629714fe90783a2da34b6a72046f62f123c08d3ae85577416438c81420812","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"04f262e515eff391723a4f4aab4135d75215b6082680c599d6430dabd60d12a3","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"s report:\n      Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled (Spec Runtime: 1h29m59.817s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.817s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n          At [By Step] creating the HCP cluster (Step Runtime: 40m59.377s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:76\n\n          Spec Goroutine\n          goroutine 124 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc000219270}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc000219270}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc000309650?}, {{0x827b848?, 0xc000ed7300?}, 0xc000a45c30?}, 0xc00111d7e0, {0xc000df20d8, 0x15}, {0x786569b, 0x14}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func24.2({0x7f4080059c60, 0xc000f06ed0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:77\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000f06ed0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 121\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83 @ 03/18/26 20:20:42.433\n  \"ts\"=\"2026-03-18 20:20:42.433473\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:42.433670\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-xs77kz\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\"\n  \"ts\"=\"2026-03-18 20:20:43.529045\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-xs77kz\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:44.451510\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.451656\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.451753\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:44.453327\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-29\"\n  \"ts\"=\"2026-03-18 20:20:44.453420\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\"\n  \"ts\"=\"2026-03-18 20:20:45.097298\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-29\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:45.644412\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-03-18 20:20:46.080042\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-29\" \"deployment\"=\"msi-deployment-29\"\n  \"ts\"=\"2026-03-18 20:20:46.547007\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\" \"deployment\"=\"customer-infra-np-autoscale-cluster-xs77kz\"\n  \"ts\"=\"2026-03-18 20:20:47.609960\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:47.610258\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:47.610426\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ns report:\n      Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled (Spec Runtime: 1h29m59.817s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.817s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n          At [By Step] creating the HCP cluster (Step Runtime: 40m59.377s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:76\n\n          Spec Goroutine\n          goroutine 124 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc000219270}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc000219270}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc000309650?}, {{0x827b848?, 0xc000ed7300?}, 0xc000a45c30?}, 0xc00111d7e0, {0xc000df20d8, 0x15}, {0x786569b, 0x14}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func24.2({0x7f4080059c60, 0xc000f06ed0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:77\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000f06ed0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 121\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83 @ 03/18/26 20:20:42.433\n  \"ts\"=\"2026-03-18 20:20:42.433473\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:42.433670\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-xs77kz\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\"\n  \"ts\"=\"2026-03-18 20:20:43.529045\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-xs77kz\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:44.451510\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.451656\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.451753\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:44.453327\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-29\"\n  \"ts\"=\"2026-03-18 20:20:44.453420\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\"\n  \"ts\"=\"2026-03-18 20:20:45.097298\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-29\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:45.644412\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-03-18 20:20:46.080042\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-29\" \"deployment\"=\"msi-deployment-29\"\n  \"ts\"=\"2026-03-18 20:20:46.547007\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\" \"deployment\"=\"customer-infra-np-autoscale-cluster-xs77kz\"\n  \"ts\"=\"2026-03-18 20:20:47.609960\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:47.610258\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:47.610426\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-kmxg9d\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"31c6c6a2b3828e797b0f9cdb10df166457fbb032ec5270cfd7f4f9d0e0e83750","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"9ab4a91db13dd30ad77dc8f9d54950043d4a64deb2496afeeb56e08738e2ae4f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ffb7d0\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-47h67v\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-47h67v, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-47h67v\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-47h67v, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009f5eb0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-47h67v\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ffb7d0\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-47h67v\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-47h67v, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-47h67v\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-47h67v, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009f5eb0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-47h67v\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"3c7dc5569cfe5ff69ea4669fe840e725c6a817f5e58919bd75982714831912ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"c99d59036f7f1fff1ba801885c97304b551c2b9bae05f47da2a31c778bb1f074","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009e82c0\u003e: \n    failed to create HCP cluster ea-listvbdptj: failed waiting for cluster=\"ea-listvbdptj\" in resourcegroup=\"ea-list-8zflz5\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\",\n      \"name\": \"01e981d5-e5aa-4ec0-a800-053ac6e07821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T18:51:34.435159653Z\",\n      \"endTime\": \"2026-03-18T18:54:16.669219148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-8zflz5/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listvbdptj: failed waiting for cluster=\\\"ea-listvbdptj\\\" in resourcegroup=\\\"ea-list-8zflz5\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"name\\\": \\\"01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.435159653Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:54:16.669219148Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-8zflz5/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009e82a0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listvbdptj\\\" in resourcegroup=\\\"ea-list-8zflz5\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"name\\\": \\\"01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.435159653Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:54:16.669219148Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-8zflz5/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009e82c0\u003e: \n    failed to create HCP cluster ea-listvbdptj: failed waiting for cluster=\"ea-listvbdptj\" in resourcegroup=\"ea-list-8zflz5\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\",\n      \"name\": \"01e981d5-e5aa-4ec0-a800-053ac6e07821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T18:51:34.435159653Z\",\n      \"endTime\": \"2026-03-18T18:54:16.669219148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-8zflz5/providers/Microsoft.Network/virtualNetworks/customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listvbdptj: failed waiting for cluster=\\\"ea-listvbdptj\\\" in resourcegroup=\\\"ea-list-8zflz5\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"name\\\": \\\"01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.435159653Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:54:16.669219148Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-8zflz5/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009e82a0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listvbdptj\\\" in resourcegroup=\\\"ea-list-8zflz5\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"name\\\": \\\"01e981d5-e5aa-4ec0-a800-053ac6e07821\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.435159653Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:54:16.669219148Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-8zflz5/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f6998594bca463b2a5e1f73639efa768847fc8b1c0a731d3479ddd6c6e9dd3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"2f90d20307562c426578693dbd5e815303071772232a1f884c60932779fee75c","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00158c1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-pn5hvg\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008b4560\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-pn5hvg\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008b4540\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-pn5hvg\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000c41ee0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-pn5hvg\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00158c1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-pn5hvg\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008b4560\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-pn5hvg\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008b4540\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-pn5hvg\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000c41ee0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-pn5hvg\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"4f0e1faba49b446686c3bd42924648d31c8e90135c7d646b6030bc1e1a6e5045","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"e334d9fb7eca0d1eed4a666b49b64d9bcbd22f69073e2038c77f2e7ea224be3b","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015521e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lrrfr6\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d53e40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lrrfr6\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000d53e20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lrrfr6\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000d53e00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lrrfr6\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015521e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-lrrfr6\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d53e40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lrrfr6\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000d53e20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lrrfr6\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000d53e00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-lrrfr6\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"5991040393fa1b72a9c0fb41dff7a2e70d55a8904dc0c2552903a71703939f6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"e75ccdb62f49beed391f5dd347e5e5c7994bcebd29c91502ab961493587b132c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000ac88b0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000ac88b0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"5c28b7f56602176f7faac9a632ded3acf14628a88a1f3a2591876799b312dde3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"18b732248dfe44f45d0d16ffdc383bf684496f1275d923928db8a90692e884f3","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013e04b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-vw57dt\" in resourcegroup=\"cluster-listing-fh2v8t\" to finish deleting: context canceled\n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-bbj57t\" in resourcegroup=\"cluster-listing-htksvg\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000f056a0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-vw57dt\\\" in resourcegroup=\\\"cluster-listing-fh2v8t\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000f05680\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-vw57dt\\\" in resourcegroup=\\\"cluster-listing-fh2v8t\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000f05660\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-vw57dt\\\" in resourcegroup=\\\"cluster-listing-fh2v8t\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xc001602d40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-bbj57t\\\" in resourcegroup=\\\"cluster-listing-htksvg\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001602d20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-bbj57t\\\" in resourcegroup=\\\"cluster-listing-htksvg\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc001602d00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-bbj57t\\\" in resourcegroup=\\\"cluster-listing-htksvg\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013e04b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-vw57dt\" in resourcegroup=\"cluster-listing-fh2v8t\" to finish deleting: context canceled\n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-bbj57t\" in resourcegroup=\"cluster-listing-htksvg\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000f056a0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-vw57dt\\\" in resourcegroup=\\\"cluster-listing-fh2v8t\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000f05680\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-vw57dt\\\" in resourcegroup=\\\"cluster-listing-fh2v8t\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000f05660\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-vw57dt\\\" in resourcegroup=\\\"cluster-listing-fh2v8t\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0xc001602d40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-bbj57t\\\" in resourcegroup=\\\"cluster-listing-htksvg\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001602d20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-bbj57t\\\" in resourcegroup=\\\"cluster-listing-htksvg\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc001602d00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-bbj57t\\\" in resourcegroup=\\\"cluster-listing-htksvg\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"665e67cb75f576e39aef2dce61844b58031a13cfa6ca880028c6cea294681742","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"69b9a7b9aa0c1663ed4377fc0bb0b8e8834739ace6f7a2adf8d13160b0e41e87","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e0090\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-fz7m8h\" in resourcegroup=\"rg-zstream-upgrade-4-21-9vvm8n\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-fz7m8h in resource group rg-zstream-upgrade-4-21-9vvm8n, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fz7m8h\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-9vvm8n\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-fz7m8h in resource group rg-zstream-upgrade-4-21-9vvm8n, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000572810\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-fz7m8h in resource group rg-zstream-upgrade-4-21-9vvm8n\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e0090\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-fz7m8h\" in resourcegroup=\"rg-zstream-upgrade-4-21-9vvm8n\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-fz7m8h in resource group rg-zstream-upgrade-4-21-9vvm8n, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-fz7m8h\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-9vvm8n\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-fz7m8h in resource group rg-zstream-upgrade-4-21-9vvm8n, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000572810\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-fz7m8h in resource group rg-zstream-upgrade-4-21-9vvm8n\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"7db6e9ddebb22260dc26778935f1b983b516f44987c801f2f8cce4245d5ebbdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"0fee1ffe50d09ba0325053d9147718aeacda44a421cffc4c929345acea8af22b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012a2498\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jc2htq\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000c95760\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-jc2htq\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000c95740\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-jc2htq\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000c95720\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-jc2htq\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012a2498\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jc2htq\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000c95760\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-jc2htq\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000c95740\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-jc2htq\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000c95720\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-jc2htq\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"932500bed1aaa75695481a3d0fc96fd49b9087cd75637395802c50594ce8f258","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"74f15455fbd12af320bdc57c7d07da963f02a93744cd385baaa3f138c8e7ac25","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001119d40\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-c4hrv4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-c4hrv4, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-c4hrv4\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-c4hrv4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009660b0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-c4hrv4\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001119d40\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-c4hrv4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-c4hrv4, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-c4hrv4\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-c4hrv4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009660b0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-c4hrv4\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}},{"row_id":"b5090be0ea303ecd3fbfcb4dc3c7d224ab01097eac699f052ac154f0c78d5f30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"d8fa07d997406a92090c6e75c4c9384140b21b393a8ca7d33f61a99eb5e48959","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0011a2a00\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-796vrt\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\",\n      \"name\": \"f7e35d02-2fd8-4915-a123-3030870ba152\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T18:51:34.793318672Z\",\n      \"endTime\": \"2026-03-18T19:06:34.464660362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'arm64-vm-customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-796vrt/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-796vrt\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"name\\\": \\\"f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.793318672Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T19:06:34.464660362Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'arm64-vm-customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-796vrt/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0011a29e0\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-796vrt\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"name\\\": \\\"f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.793318672Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T19:06:34.464660362Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'arm64-vm-customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-796vrt/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additiona...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0011a2a00\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-796vrt\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\",\n      \"name\": \"f7e35d02-2fd8-4915-a123-3030870ba152\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T18:51:34.793318672Z\",\n      \"endTime\": \"2026-03-18T19:06:34.464660362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'arm64-vm-customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-796vrt/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-796vrt\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"name\\\": \\\"f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.793318672Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T19:06:34.464660362Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'arm64-vm-customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-796vrt/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0011a29e0\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-796vrt\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"name\\\": \\\"f7e35d02-2fd8-4915-a123-3030870ba152\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:34.793318672Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T19:06:34.464660362Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'arm64-vm-customer-vnet' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-796vrt/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additiona...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bc0c81d7188b4cf69be6865efc63ff1dac76e1242ea440dcb7c70f4d23814c11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"41b3212b2001e559d2853044295b852b6d1719b3b77c7d798b6e8169df611150","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"re's a current progress report:\n      Customer should be able to update node pool labels and taints (Spec Runtime: 1h29m59.927s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n        In [It] (Node Runtime: 1h29m59.927s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n          At [By Step] creating the HCP cluster (Step Runtime: 21m9.913s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:83\n\n          Spec Goroutine\n          goroutine 44 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc000806500}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc000806500}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc0005ec150?}, {{0x827b848?, 0xc000cbe300?}, 0xc000a971d0?}, 0xc00009b4f0, {0xc001307e20, 0x1a}, {0x78794ae, 0x18}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func25.1({0x7f72e04d1460, 0xc000b19e60})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:84\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000b19e60?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91 @ 03/18/26 20:20:44.776\n  \"ts\"=\"2026-03-18 20:20:44.776726\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:44.776825\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-xqhhfh\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\"\n  \"ts\"=\"2026-03-18 20:20:45.909847\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-xqhhfh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:46.664043\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.664206\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.664290\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:46.666099\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-60\"\n  \"ts\"=\"2026-03-18 20:20:46.666112\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\"\n  \"ts\"=\"2026-03-18 20:20:47.286227\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-60\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:47.883058\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-03-18 20:20:48.248972\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-60\" \"deployment\"=\"msi-deployment-60\"\n  \"ts\"=\"2026-03-18 20:20:48.769750\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-xqhhfh\"\n  \"ts\"=\"2026-03-18 20:20:49.765203\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:49.765412\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:49.765637\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nre's a current progress report:\n      Customer should be able to update node pool labels and taints (Spec Runtime: 1h29m59.927s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n        In [It] (Node Runtime: 1h29m59.927s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n          At [By Step] creating the HCP cluster (Step Runtime: 21m9.913s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:83\n\n          Spec Goroutine\n          goroutine 44 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc000806500}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc000806500}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc0005ec150?}, {{0x827b848?, 0xc000cbe300?}, 0xc000a971d0?}, 0xc00009b4f0, {0xc001307e20, 0x1a}, {0x78794ae, 0x18}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func25.1({0x7f72e04d1460, 0xc000b19e60})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:84\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000b19e60?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91 @ 03/18/26 20:20:44.776\n  \"ts\"=\"2026-03-18 20:20:44.776726\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:44.776825\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-xqhhfh\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\"\n  \"ts\"=\"2026-03-18 20:20:45.909847\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-xqhhfh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:46.664043\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.664206\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.664290\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:46.666099\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-60\"\n  \"ts\"=\"2026-03-18 20:20:46.666112\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\"\n  \"ts\"=\"2026-03-18 20:20:47.286227\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-60\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:47.883058\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-03-18 20:20:48.248972\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-60\" \"deployment\"=\"msi-deployment-60\"\n  \"ts\"=\"2026-03-18 20:20:48.769750\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-xqhhfh\"\n  \"ts\"=\"2026-03-18 20:20:49.765203\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:49.765412\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:49.765637\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-tgzmm2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"c377423dce731d747c871b79a4661bb3b122ef07cf8ba7423dbaf7ac38ecaf81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"907bf5fdd01c6d1102f420f3f9364f5fb0b9eff72ac3495d7913cb7f9d50d607","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"stomer should be able to update nodepool replicas and autoscaling (Spec Runtime: 1h29m59.937s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35\n        In [It] (Node Runtime: 1h29m59.937s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 21m20.215s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:75\n\n          Spec Goroutine\n          goroutine 96 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc0007e4640}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc0007e4640}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc0006d22a0?}, {{0x827b848?, 0xc000814500?}, 0xc001606a30?}, 0xc001287600, {0xc000b40320, 0x1c}, {0x7888076, 0x1b}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f9f24269e60, 0xc000960f90})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:76\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000960f90?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 94\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82 @ 03/18/26 20:20:45.017\n  \"ts\"=\"2026-03-18 20:20:45.017585\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:45.017722\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-d7c8dd\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\"\n  \"ts\"=\"2026-03-18 20:20:46.051413\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-update-nodes-hcp-cluster-d7c8dd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:46.935614\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.935743\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.935831\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:46.937287\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-62\"\n  \"ts\"=\"2026-03-18 20:20:46.937308\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\"\n  \"ts\"=\"2026-03-18 20:20:47.628469\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-62\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:48.065445\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\" \"deployment\"=\"pooledNonMsiScopedAssignments-w3k44uxsb3qey\"\n  \"ts\"=\"2026-03-18 20:20:48.541601\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-62\" \"deployment\"=\"msi-deployment-62\"\n  \"ts\"=\"2026-03-18 20:20:49.112141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-d7c8dd\"\n  \"ts\"=\"2026-03-18 20:20:50.054507\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:50.054711\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:50.054902\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nstomer should be able to update nodepool replicas and autoscaling (Spec Runtime: 1h29m59.937s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35\n        In [It] (Node Runtime: 1h29m59.937s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 21m20.215s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:75\n\n          Spec Goroutine\n          goroutine 96 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc0007e4640}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc0007e4640}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc0006d22a0?}, {{0x827b848?, 0xc000814500?}, 0xc001606a30?}, 0xc001287600, {0xc000b40320, 0x1c}, {0x7888076, 0x1b}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f9f24269e60, 0xc000960f90})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:76\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000960f90?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 94\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82 @ 03/18/26 20:20:45.017\n  \"ts\"=\"2026-03-18 20:20:45.017585\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:45.017722\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-d7c8dd\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\"\n  \"ts\"=\"2026-03-18 20:20:46.051413\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-update-nodes-hcp-cluster-d7c8dd\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:46.935614\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.935743\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4d7111da7878873ab737ec2a993f733d74239d4c919cccd4b38c36a47c0a2edd.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:46.935831\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:46.937287\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-62\"\n  \"ts\"=\"2026-03-18 20:20:46.937308\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\"\n  \"ts\"=\"2026-03-18 20:20:47.628469\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-62\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:48.065445\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\" \"deployment\"=\"pooledNonMsiScopedAssignments-w3k44uxsb3qey\"\n  \"ts\"=\"2026-03-18 20:20:48.541601\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-62\" \"deployment\"=\"msi-deployment-62\"\n  \"ts\"=\"2026-03-18 20:20:49.112141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\" \"deployment\"=\"customer-infra-np-update-nodes-hcp-cluster-d7c8dd\"\n  \"ts\"=\"2026-03-18 20:20:50.054507\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:50.054711\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:50.054902\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"nodepool-update-nodes-nj695s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"c3acd3f6be9533aa353f4260dcf7625ee5414058bd5fde607bbb7f8a56ec867a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"0a420bd1152df546566584c11dcc16c9920790cd81443aa1f1ba125b60fa442b","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"si-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000ab4b98, {0xc000c7f860?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000bdea88, {0x0, 0x0}, {0xb38f160, 0x0, 0x0}, {0xc00007c064, 0x2a}, 0x0, {0xc000c7f840, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007c064, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2a5da65?, 0xc000785ec0?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8266018, 0xc000c645a0}, 0xc000533e00, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 24 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000533e00?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 87\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95 @ 03/18/26 20:20:42.018\n  \"ts\"=\"2026-03-18 20:20:42.018432\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:42.018540\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-vxcbt8-vxf8pq\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\"\n  \"ts\"=\"2026-03-18 20:20:43.160580\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-zstream-4-19-vxcbt8-vxf8pq\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:43.898952\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:43.899124\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:43.899218\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:43.900635\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-100\"\n  \"ts\"=\"2026-03-18 20:20:43.900698\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\"\n  \"ts\"=\"2026-03-18 20:20:44.490292\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-100\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:45.076813\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\" \"deployment\"=\"pooledNonMsiScopedAssignments-mq76aepq5vgly\"\n  \"ts\"=\"2026-03-18 20:20:45.542169\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-100\" \"deployment\"=\"msi-deployment-100\"\n  \"ts\"=\"2026-03-18 20:20:46.026274\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-vxcbt8-vxf8pq\"\n  \"ts\"=\"2026-03-18 20:20:46.979701\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:46.979869\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:46.980038\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000ab4b98, {0xc000c7f860?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000bdea88, {0x0, 0x0}, {0xb38f160, 0x0, 0x0}, {0xc00007c064, 0x2a}, 0x0, {0xc000c7f840, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007c064, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2a5da65?, 0xc000785ec0?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8266018, 0xc000c645a0}, 0xc000533e00, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 24 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000533e00?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 87\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95 @ 03/18/26 20:20:42.018\n  \"ts\"=\"2026-03-18 20:20:42.018432\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:42.018540\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-vxcbt8-vxf8pq\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\"\n  \"ts\"=\"2026-03-18 20:20:43.160580\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-zstream-4-19-vxcbt8-vxf8pq\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:43.898952\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:43.899124\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:43.899218\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:43.900635\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-100\"\n  \"ts\"=\"2026-03-18 20:20:43.900698\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\"\n  \"ts\"=\"2026-03-18 20:20:44.490292\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-100\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:45.076813\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\" \"deployment\"=\"pooledNonMsiScopedAssignments-mq76aepq5vgly\"\n  \"ts\"=\"2026-03-18 20:20:45.542169\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-100\" \"deployment\"=\"msi-deployment-100\"\n  \"ts\"=\"2026-03-18 20:20:46.026274\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-vxcbt8-vxf8pq\"\n  \"ts\"=\"2026-03-18 20:20:46.979701\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:46.979869\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:46.980038\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-bck7b7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"dfa8281573389150965725f56238d05ccce5faa69d929c0b793f32d1df9b12a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"2c27fa56b1b82efd3d18f6773882f4b36efcb474c4a09ab407e4af47fa2676da","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"progress report:\n      Customer should respect cluster-wide node limits with nodepool autoscaling (Spec Runtime: 1h29m59.735s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n        In [It] (Node Runtime: 1h29m59.735s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n          At [By Step] creating cluster with MaxNodesTotal limit (Step Runtime: 41m56.808s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:143\n\n          Spec Goroutine\n          goroutine 96 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc000e9b040}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc000e9b040}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc000ac4d90?}, {{0x827b848?, 0xc000e95880?}, 0xc001680d20?}, 0xc0010ef810, {0xc000aeb230, 0x16}, {0x785ae5f, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func24.3({0x7f676f6cc2a0, 0xc000ea3ef0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:160\n                | Expect(err).NotTo(HaveOccurred())\n                | \n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000ea3ef0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 93\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166 @ 03/18/26 20:20:42.506\n  \"ts\"=\"2026-03-18 20:20:42.507185\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:42.507302\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-node-limit-cluster-q2pqxn\" \"resourceGroup\"=\"node-limit-test-dbckdp\"\n  \"ts\"=\"2026-03-18 20:20:43.677122\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-node-limit-cluster-q2pqxn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:44.490455\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.490632\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.490759\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:44.493288\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-148\"\n  \"ts\"=\"2026-03-18 20:20:44.493345\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"node-limit-test-dbckdp\"\n  \"ts\"=\"2026-03-18 20:20:45.137419\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:45.646482\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-dbckdp\" \"deployment\"=\"pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\"\n  \"ts\"=\"2026-03-18 20:20:46.128381\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-148\" \"deployment\"=\"msi-deployment-148\"\n  \"ts\"=\"2026-03-18 20:20:46.632160\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-dbckdp\" \"deployment\"=\"customer-infra-node-limit-cluster-q2pqxn\"\n  \"ts\"=\"2026-03-18 20:20:47.650775\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:47.650925\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:47.651309\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"node-limit-test-dbckdp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nprogress report:\n      Customer should respect cluster-wide node limits with nodepool autoscaling (Spec Runtime: 1h29m59.735s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n        In [It] (Node Runtime: 1h29m59.735s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n          At [By Step] creating cluster with MaxNodesTotal limit (Step Runtime: 41m56.808s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:143\n\n          Spec Goroutine\n          goroutine 96 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8266018, 0xc000e9b040}, 0x783ee71?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8299000, {0x8266018, 0xc000e9b040}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8266088?, 0xc000ac4d90?}, {{0x827b848?, 0xc000e95880?}, 0xc001680d20?}, 0xc0010ef810, {0xc000aeb230, 0x16}, {0x785ae5f, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:658\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x781f451, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func24.3({0x7f676f6cc2a0, 0xc000ea3ef0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:160\n                | Expect(err).NotTo(HaveOccurred())\n                | \n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b920?, 0xc000ea3ef0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 93\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166 @ 03/18/26 20:20:42.506\n  \"ts\"=\"2026-03-18 20:20:42.507185\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 20:20:42.507302\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-node-limit-cluster-q2pqxn\" \"resourceGroup\"=\"node-limit-test-dbckdp\"\n  \"ts\"=\"2026-03-18 20:20:43.677122\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-node-limit-cluster-q2pqxn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 20:20:44.490455\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.490632\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-18 20:20:44.490759\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:44.493288\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-148\"\n  \"ts\"=\"2026-03-18 20:20:44.493345\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"node-limit-test-dbckdp\"\n  \"ts\"=\"2026-03-18 20:20:45.137419\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 20:20:45.646482\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-dbckdp\" \"deployment\"=\"pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\"\n  \"ts\"=\"2026-03-18 20:20:46.128381\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-148\" \"deployment\"=\"msi-deployment-148\"\n  \"ts\"=\"2026-03-18 20:20:46.632160\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-dbckdp\" \"deployment\"=\"customer-infra-node-limit-cluster-q2pqxn\"\n  \"ts\"=\"2026-03-18 20:20:47.650775\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 20:20:47.650925\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 20:20:47.651309\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"node-limit-test-dbckdp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e03344324dc8a5086c25afefacd22bd3234ea73849d49ba556c50146e1a6d87a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034331286524399616","occurred_at":"2026-03-18T18:09:09Z","failed_at":"e2e","signature_id":"94978e59d6f4b80c27689301ad038f2f9531ca6c657a3a7b57dbde69300d9392","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0016e5560\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-njw9zk\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\",\n      \"name\": \"ffea3a4c-4428-4297-adbc-cee4edb4dc69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T18:51:36.043041035Z\",\n      \"endTime\": \"2026-03-18T18:52:41.369770418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-njw9zk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-njw9zk\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"name\\\": \\\"ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:36.043041035Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:52:41.369770418Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-njw9zk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0016e5540\u003e{\n            msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-njw9zk\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"name\\\": \\\"ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:36.043041035Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:52:41.369770418Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-njw9zk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0016e5560\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-njw9zk\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\",\n      \"name\": \"ffea3a4c-4428-4297-adbc-cee4edb4dc69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T18:51:36.043041035Z\",\n      \"endTime\": \"2026-03-18T18:52:41.369770418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-njw9zk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-njw9zk\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"name\\\": \\\"ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:36.043041035Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:52:41.369770418Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-njw9zk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0016e5540\u003e{\n            msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-njw9zk\\\" to finish creating: GET https://rp.j4399616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"name\\\": \\\"ffea3a4c-4428-4297-adbc-cee4edb4dc69\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T18:51:36.043041035Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T18:52:41.369770418Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-njw9zk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":25,"failure_pattern_ids":["15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034328701339963392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4425,"pr_state":"closed","pr_sha":"64a2f7a8b65b208052e3c620a9c0db2c86c6a372","final_merged_sha":"855eea85f3052237fc8528c6d280c0ac41ee4307","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T17:58:52Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"23e145b9f8616aee37cc8d4bd3cb1d585c8238af621bd782a51b848c5a87e014","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034328701339963392","occurred_at":"2026-03-18T17:58:52Z","failed_at":"e2e","signature_id":"01145bd78e0c98c5f931fc940d5115932f2777197c1411d03d972ebae7faa6ea","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00108f610\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00108f610\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"2fc05ea39c72f83c72e9445b022006fdd3bec5c5c8db45cf51f31420eef8ece5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034328701339963392","occurred_at":"2026-03-18T17:58:52Z","failed_at":"e2e","signature_id":"920e8dee6e3fb05450e1242141bc6c820b3d73307f365f13d87fbabdf4f18adb","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ffa228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00173af40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tbxqkj\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j5ckd5\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00173af20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tbxqkj\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j5ckd5\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00173af00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tbxqkj\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j5ckd5\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb1af0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ffa228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00173af40\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tbxqkj\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j5ckd5\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00173af20\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tbxqkj\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j5ckd5\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00173af00\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tbxqkj\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-j5ckd5\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb1af0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"b260f19747269a57797e8866fa63058186931369c8b50334568ac335ea3395b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4425/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034328701339963392","occurred_at":"2026-03-18T17:58:52Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034327444416106496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4516,"pr_state":"open","pr_sha":"3e72d4002cd7f32245bb8bb926429522f6a7f89e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T17:53:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4b1d975526f5bf3be8b203ddc55dca36e20205251fcee08b40f5153dc55654b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034327444416106496","occurred_at":"2026-03-18T17:53:53Z","failed_at":"e2e","signature_id":"3cdba9ad2df367584ad79c9f85944beda2d9049b2488ea0eab35317eb6223b19","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00140a820\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00140a820\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034325389999869952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"c84eed08362bdc95a6bee238819cb7de596fcb69","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T17:45:42Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0da3f6246df5cfc7b50cdcadc1c465082560de2ce99cc80ca0463a00861db325","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034325389999869952","occurred_at":"2026-03-18T17:45:42Z","failed_at":"e2e","signature_id":"9fb190ff5cd269f80ccb9ca598b153b98669521f141a8bd2c4af2019993b531a","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"upResourceGroups.func1({0x7f93243730a0, 0xc001695e60})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:294 +0x14d fp=0xc000b77fc0 sp=0xc000b77f28 pc=0x4ec572d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:303 +0x28 fp=0xc000b77fe0 sp=0xc000b77fc0 pc=0x4ec55a8\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b77fe8 sp=0xc000b77fe0 pc=0x2ad7921\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 1224\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:287 +0xc5\n\ngoroutine 1449 gp=0xc000e03180 m=nil [IO wait]:\nruntime.gopark(0x0?, 0x2424ab48?, 0x50?, 0xf7?, 0xb?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000cf720 sp=0xc0000cf700 pc=0x2acf28e\nruntime.netpollblock(0x2b38d58?, 0x2a5b046?, 0x0?)\n\t/usr/local/go/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000cf758 sp=0xc0000cf720 pc=0x2a8ee97\ninternal/poll.runtime_pollWait(0x7f9327dbc000, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:351 +0x85 fp=0xc0000cf778 sp=0xc0000cf758 pc=0x2acdb25\ninternal/poll.(*pollDesc).wait(0xc000294700?, 0xc000e1a000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000cf7a0 sp=0xc0000cf778 pc=0x2b5ba87\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000294700, {0xc000e1a000, 0x5500, 0x5500})\n\t/usr/local/go/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000cf838 sp=0xc0000cf7a0 pc=0x2b5ceb9\nnet.(*netFD).Read(0xc000294700, {0xc000e1a000?, 0xc0000cf948?, 0x2a5895c?})\n\t/usr/local/go/src/net/fd_posix.go:68 +0x25 fp=0xc0000cf880 sp=0xc0000cf838 pc=0x2cf1d65\nnet.(*conn).Read(0xc000ef60a0, {0xc000e1a000?, 0x7f932436f9e8?, 0x7f936e9a3a78?})\n\t/usr/local/go/src/net/net.go:196 +0x45 fp=0xc0000cf8c8 sp=0xc0000cf880 pc=0x2d03e05\ncrypto/tls.(*atLeastReader).Read(0xc00104e3d8, {0xc000e1a000?, 0x784a078?, 0xe?})\n\t/usr/local/go/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000cf910 sp=0xc0000cf8c8 pc=0x2d6f6bb\nbytes.(*Buffer).ReadFrom(0xc00091e2a8, {0x81cfac0, 0xc00104e3d8})\n\t/usr/local/go/src/bytes/buffer.go:217 +0x98 fp=0xc0000cf968 sp=0xc0000cf910 pc=0x2b99a38\ncrypto/tls.(*Conn).readFromUntil(0xc00091e008, {0x81d04c0, 0xc000ef60a0}, 0xc0000cf9d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc0000cf9a0 sp=0xc0000cf968 pc=0x2d6f89e\ncrypto/tls.(*Conn).readRecordOrCCS(0xc00091e008, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000cfc18 sp=0xc0000cf9a0 pc=0x2d6c91b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc00091e008, {0xc000934000, 0x1000, 0x31aed60?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000cfc80 sp=0xc0000cfc18 pc=0x2d73205\nbufio.(*Reader).Read(0xc001751b60, {0xc0014442e4, 0x9, 0x31bce0e?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc0000cfcb8 sp=0xc0000cfc80 pc=0x2c351f7\nio.ReadAtLeast({0x81cf0a0, 0xc001751b60}, {0xc0014442e4, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc0000cfd00 sp=0xc0000cfcb8 pc=0x2b29aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014442e4, 0x9, 0xc000000011?}, {0x81cf0a0?, 0xc001751b60?})\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:250 +0x65 fp=0xc0000cfd50 sp=0xc0000cfd00 pc=0x31a5b25\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014442a0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:513 +0x6b fp=0xc0000cfdd8 sp=0xc0000cfd50 pc=0x31a66eb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014442a0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:572 +0x18 fp=0xc0000cfe18 sp=0xc0000cfdd8 pc=0x31a6e58\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc0000cffa8)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2254 +0xca fp=0xc0000cff60 sp=0xc0000cfe18 pc=0x31b9d4a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000d048c0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2123 +0x52 fp=0xc0000cffc8 sp=0xc0000cff60 pc=0x31b90d2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0x25 fp=0xc0000cffe0 sp=0xc0000cffc8 pc=0x31b2865\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000cffe8 sp=0xc0000cffe0 pc=0x2ad7921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 1448\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x2ad9743\nrdx    0x0\nrdi    0xb367af0\nrsi    0x80\nrbp    0x7ffde5778ed0\nrsp    0x7ffde5778e88\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0xc0000a7208\nr13    0x1\nr14    0xb365a00\nr15    0x1\nrip    0x2ad9741\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nupResourceGroups.func1({0x7f93243730a0, 0xc001695e60})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:294 +0x14d fp=0xc000b77fc0 sp=0xc000b77f28 pc=0x4ec572d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:303 +0x28 fp=0xc000b77fe0 sp=0xc000b77fc0 pc=0x4ec55a8\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b77fe8 sp=0xc000b77fe0 pc=0x2ad7921\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 1224\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:287 +0xc5\n\ngoroutine 1449 gp=0xc000e03180 m=nil [IO wait]:\nruntime.gopark(0x0?, 0x2424ab48?, 0x50?, 0xf7?, 0xb?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000cf720 sp=0xc0000cf700 pc=0x2acf28e\nruntime.netpollblock(0x2b38d58?, 0x2a5b046?, 0x0?)\n\t/usr/local/go/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000cf758 sp=0xc0000cf720 pc=0x2a8ee97\ninternal/poll.runtime_pollWait(0x7f9327dbc000, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:351 +0x85 fp=0xc0000cf778 sp=0xc0000cf758 pc=0x2acdb25\ninternal/poll.(*pollDesc).wait(0xc000294700?, 0xc000e1a000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000cf7a0 sp=0xc0000cf778 pc=0x2b5ba87\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000294700, {0xc000e1a000, 0x5500, 0x5500})\n\t/usr/local/go/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000cf838 sp=0xc0000cf7a0 pc=0x2b5ceb9\nnet.(*netFD).Read(0xc000294700, {0xc000e1a000?, 0xc0000cf948?, 0x2a5895c?})\n\t/usr/local/go/src/net/fd_posix.go:68 +0x25 fp=0xc0000cf880 sp=0xc0000cf838 pc=0x2cf1d65\nnet.(*conn).Read(0xc000ef60a0, {0xc000e1a000?, 0x7f932436f9e8?, 0x7f936e9a3a78?})\n\t/usr/local/go/src/net/net.go:196 +0x45 fp=0xc0000cf8c8 sp=0xc0000cf880 pc=0x2d03e05\ncrypto/tls.(*atLeastReader).Read(0xc00104e3d8, {0xc000e1a000?, 0x784a078?, 0xe?})\n\t/usr/local/go/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000cf910 sp=0xc0000cf8c8 pc=0x2d6f6bb\nbytes.(*Buffer).ReadFrom(0xc00091e2a8, {0x81cfac0, 0xc00104e3d8})\n\t/usr/local/go/src/bytes/buffer.go:217 +0x98 fp=0xc0000cf968 sp=0xc0000cf910 pc=0x2b99a38\ncrypto/tls.(*Conn).readFromUntil(0xc00091e008, {0x81d04c0, 0xc000ef60a0}, 0xc0000cf9d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc0000cf9a0 sp=0xc0000cf968 pc=0x2d6f89e\ncrypto/tls.(*Conn).readRecordOrCCS(0xc00091e008, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000cfc18 sp=0xc0000cf9a0 pc=0x2d6c91b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc00091e008, {0xc000934000, 0x1000, 0x31aed60?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000cfc80 sp=0xc0000cfc18 pc=0x2d73205\nbufio.(*Reader).Read(0xc001751b60, {0xc0014442e4, 0x9, 0x31bce0e?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc0000cfcb8 sp=0xc0000cfc80 pc=0x2c351f7\nio.ReadAtLeast({0x81cf0a0, 0xc001751b60}, {0xc0014442e4, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc0000cfd00 sp=0xc0000cfcb8 pc=0x2b29aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014442e4, 0x9, 0xc000000011?}, {0x81cf0a0?, 0xc001751b60?})\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:250 +0x65 fp=0xc0000cfd50 sp=0xc0000cfd00 pc=0x31a5b25\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014442a0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:513 +0x6b fp=0xc0000cfdd8 sp=0xc0000cfd50 pc=0x31a66eb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014442a0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:572 +0x18 fp=0xc0000cfe18 sp=0xc0000cfdd8 pc=0x31a6e58\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc0000cffa8)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2254 +0xca fp=0xc0000cff60 sp=0xc0000cfe18 pc=0x31b9d4a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000d048c0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2123 +0x52 fp=0xc0000cffc8 sp=0xc0000cff60 pc=0x31b90d2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0x25 fp=0xc0000cffe0 sp=0xc0000cffc8 pc=0x31b2865\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000cffe8 sp=0xc0000cffe0 pc=0x2ad7921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 1448\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x2ad9743\nrdx    0x0\nrdi    0xb367af0\nrsi    0x80\nrbp    0x7ffde5778ed0\nrsp    0x7ffde5778e88\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0xc0000a7208\nr13    0x1\nr14    0xb365a00\nr15    0x1\nrip    0x2ad9741\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"cb73c46cf5d6c7207a3236ad5ccc6de7699feb536406608c5d81e269c9141ade","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034325389999869952","occurred_at":"2026-03-18T17:45:42Z","failed_at":"e2e","signature_id":"2cdd986c4318e1e35c57e21c1796c415db7c728994a32e4485764088410db46c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00020d290\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00020d290\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034316497634988032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3907,"pr_state":"closed","pr_sha":"500db4448b402960862e86f32ab3e4102071222d","final_merged_sha":"e9ca15da6d935a378fb165189c2441a771314f73","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T17:10:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bb4b519d7dbc370beff9e38543c2bddcd70280f50ea9985b88f7b0fd5919c045","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3907/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034316497634988032","occurred_at":"2026-03-18T17:10:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034315696216739840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4515,"pr_state":"closed","pr_sha":"fd99db312afd8c50e15b0c5fafbb2271e2bfe325","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T17:07:12Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"4b67154d5719a6f3d5d77c9acea8b19d101fe050531ea1fa523ace5e03a9b766","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034315696216739840","occurred_at":"2026-03-18T17:07:12Z","failed_at":"e2e","signature_id":"bec00857dd1a0ff28f10cf73e13776bc41569699a330155bb6ec9d6eca29a802","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00057ccc0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-fclq4l\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-fclq4l, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-fclq4l\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-fclq4l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0002fae20\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-fclq4l\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00057ccc0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-fclq4l\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-fclq4l, error: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-fclq4l\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-fclq4l, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0002fae20\u003e{\n                s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-fclq4l\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-dd4h2h\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dd4h2h, error: context deadline exceeded"}},{"row_id":"9222cfe08b593ad34a448e3caa8b41c77be57dee24bda1dacfb9b08b33aa4194","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034315696216739840","occurred_at":"2026-03-18T17:07:12Z","failed_at":"e2e","signature_id":"db65183c85f7611ff70a4d10ec94e16b609e70cc91b03d137f354790d7f14b53","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:276]: Timed out after 300.007s.\nExpected\n    \u003cv1.PodPhase\u003e: Running\nto equal\n    \u003cv1.PodPhase\u003e: Succeeded\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:276]: Timed out after 300.007s.\nExpected\n    \u003cv1.PodPhase\u003e: Running\nto equal\n    \u003cv1.PodPhase\u003e: Succeeded","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b2294f64176300144ce8b3080430bffa51c6b57bf80df075d489f3117ec76ca","failure_pattern":"Customer should be able to use workload identity via the cluster OIDC issuer URL: Timed out after \u003cduration\u003es.","search_query":"Timed out after 300.007s."}},{"row_id":"fe2001d0e3c4d5d2eceabe31f86c072370f304b34dc0bd81bf36462f5716314e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034315696216739840","occurred_at":"2026-03-18T17:07:12Z","failed_at":"e2e","signature_id":"d9e90ddfd0b4505d153ad4e4b9b39ab48ae0456df8dd173d448f66d0fbd42dc9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0008a7e90\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0008a7e90\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","7b2294f64176300144ce8b3080430bffa51c6b57bf80df075d489f3117ec76ca","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034310420222709760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4514,"pr_state":"closed","pr_sha":"f89aa7097b020f257f73dd6b15c1134deaf3900d","final_merged_sha":"61e704d766a95da5a14cf10c199fb79215449bb4","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T16:46:07Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2e8a22fedf9d15893eace2fb9d05f4498297bfb9978164fcfe6bb1d02130d0ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034310420222709760","occurred_at":"2026-03-18T16:46:07Z","failed_at":"e2e","signature_id":"42b34233a4b3024dac5c7339bfa519cd913f4847ec10f84d3c601299560257e6","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017c61b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-p84c9b\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001663640\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-p84c9b\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001663620\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-p84c9b\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc001663600\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-p84c9b\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017c61b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-p84c9b\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001663640\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-p84c9b\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc001663620\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-p84c9b\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc001663600\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-p84c9b\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"cee23386dfdde743d1878747e8f133320d1c5154da247839aa6b079e5509ca86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034310420222709760","occurred_at":"2026-03-18T16:46:07Z","failed_at":"e2e","signature_id":"5ad9450551d45631fef11d107697175fd565df3b4d628166c40d9d8eefed6b8b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0002c1610\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.006s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0002c1610\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"f5d811eab57b3030d3c9be2b60c3dc226a47c91529ef91492f10e93645920b0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034310420222709760","occurred_at":"2026-03-18T16:46:07Z","failed_at":"e2e","signature_id":"367f7022275d882175983fecb7d61028a83a0609f64c128d6cc2af5a7882849a","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000012618\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-zwswk7\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008e1600\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-zwswk7\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008e15e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-zwswk7\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0008e15c0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-zwswk7\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000012618\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-zwswk7\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008e1600\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-zwswk7\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008e15e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-zwswk7\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0008e15c0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-zwswk7\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034306786592296960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4444,"pr_state":"closed","pr_sha":"092800808fd635f2494b3d3e1993603df989962b","final_merged_sha":"092800808fd635f2494b3d3e1993603df989962b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T16:31:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"878142c33a007d69e4973dc391940aaa80c46fb622b9ee405e56546aa35dcbd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034306786592296960","occurred_at":"2026-03-18T16:31:47Z","failed_at":"e2e","signature_id":"de3ee4681814121c69c9ac6431ebdef2646f68187fced0e7f70752fbc20cc35a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000e5a2f0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000e5a2f0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034302062623199232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4512,"pr_state":"closed","pr_sha":"ca7bbc980a35e3fc6dc73742e2b24e1d36f1178a","final_merged_sha":"ca7bbc980a35e3fc6dc73742e2b24e1d36f1178a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T16:13:01Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5057c5541bf790473c5c1e5a402d71c68efeb0e151445f6737b7d3544cc2396a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034302062623199232","occurred_at":"2026-03-18T16:13:01Z","failed_at":"e2e","signature_id":"58061b9eb8607943856150e28fa4af63766829a100d20315869efcf933ba0b28","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001177610\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001177610\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"ea8da20e207ac53316072ace415637991c32563b8ee02327d6ae62d6951f352c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034302062623199232","occurred_at":"2026-03-18T16:13:01Z","failed_at":"e2e","signature_id":"7e641226f90ccc9d8691d11b1d8fece265254da825665e56fef91c6488ac05db","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00099e900\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-sj46dk\" to finish creating: GET https://rp.j3199232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\",\n      \"name\": \"621a39ba-9923-42f5-b1ef-878422819b7f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T16:39:27.380883412Z\",\n      \"endTime\": \"2026-03-18T16:41:27.012428471Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-sj46dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-sj46dk\\\" to finish creating: GET https://rp.j3199232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"name\\\": \\\"621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:39:27.380883412Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:41:27.012428471Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-sj46dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00099e8e0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-sj46dk\\\" to finish creating: GET https://rp.j3199232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"name\\\": \\\"621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:39:27.380883412Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:41:27.012428471Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-sj46dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00099e900\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-sj46dk\" to finish creating: GET https://rp.j3199232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\",\n      \"name\": \"621a39ba-9923-42f5-b1ef-878422819b7f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T16:39:27.380883412Z\",\n      \"endTime\": \"2026-03-18T16:41:27.012428471Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-sj46dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-sj46dk\\\" to finish creating: GET https://rp.j3199232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"name\\\": \\\"621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:39:27.380883412Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:41:27.012428471Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-sj46dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00099e8e0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-sj46dk\\\" to finish creating: GET https://rp.j3199232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"name\\\": \\\"621a39ba-9923-42f5-b1ef-878422819b7f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:39:27.380883412Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:41:27.012428471Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-sj46dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4511/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034301979622117376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4511,"pr_state":"closed","pr_sha":"39d25fe41a7f321f042998103d74578dce0f5bc4","final_merged_sha":"39d25fe41a7f321f042998103d74578dce0f5bc4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T16:12:42Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"16ce820c4e07a1fded32f834c53532dd428436c1bdefd8f451be1a06dc19305e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4511/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034301979622117376","occurred_at":"2026-03-18T16:12:42Z","failed_at":"e2e","signature_id":"e40ec8c4b2615a4445bebbc80b08948624929438d7fee3b9dd1f9e248d9748c4","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0009d2200\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0009d2200\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034301920981553152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"63bb89f115354070e54b6c545e5e886f2667d5b7","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T16:12:28Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"491dd243b063f14a9294d8256d52c407f328d6b62f063db2b47cdb1cf2aedd16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034301920981553152","occurred_at":"2026-03-18T16:12:28Z","failed_at":"e2e","signature_id":"67d9189f594d1b57c4e2e52cb5a2033e3d15d7c7c9fd95b51d3e5d75ac4ef9a1","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b17ba0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.008s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b17ba0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"af61ae780eb98922a8bdb4de4908670f707d4c69264d5efdd4b199f3b9e40a75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034301920981553152","occurred_at":"2026-03-18T16:12:28Z","failed_at":"e2e","signature_id":"7982ad108f2ee785d2af5bd1354c20ff0ff06f96b33d0619ce41c66092b6712d","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015b32f0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-knf62b\" in resourcegroup=\"admin-credential-lifecycle-test-xjdb4r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-knf62b in resource group admin-credential-lifecycle-test-xjdb4r, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-knf62b\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xjdb4r\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-knf62b in resource group admin-credential-lifecycle-test-xjdb4r, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001151760\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-knf62b in resource group admin-credential-lifecycle-test-xjdb4r\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015b32f0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-knf62b\" in resourcegroup=\"admin-credential-lifecycle-test-xjdb4r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-knf62b in resource group admin-credential-lifecycle-test-xjdb4r, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-knf62b\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xjdb4r\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-knf62b in resource group admin-credential-lifecycle-test-xjdb4r, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001151760\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-knf62b in resource group admin-credential-lifecycle-test-xjdb4r\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-hlgz5x\" in resourcegroup=\"admin-credential-lifecycle-test-j4scpw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-hlgz5x in resource group admin-credential-lifecycle-test-j4scpw, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["15b183e3881967176eb66fd4ef31efbd8a4611cec73c8de3f08e989dcdc536ea","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034295980936925184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4482,"pr_state":"closed","pr_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","final_merged_sha":"04f43f02c3600758a856fe0f5464f9937b913cdd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T15:48:51Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8f72c62e4130ab844b98b183db6ca95c64dbec147fc42f1eaff0df568880bffe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034295980936925184","occurred_at":"2026-03-18T15:48:51Z","failed_at":"e2e","signature_id":"2eae9e8861b9c06bd9b9446a4da20896766289ab821030ac632feb43b2542d64","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0015d0e30\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0015d0e30\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}},{"row_id":"bd776281260cc5f070e4a83027c46fe52207aef7297dce38ee2a7588b3ed14b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4482/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034295980936925184","occurred_at":"2026-03-18T15:48:51Z","failed_at":"e2e","signature_id":"e4f5cbea4855fe15e693db201c2df8087792742f821e77ecfb7c10f255b298a1","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000492920\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-vrcl27\" to finish creating: GET https://rp.j6925184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\",\n      \"name\": \"116083cc-ad76-4a20-a638-573bdc265971\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T16:29:11.853285095Z\",\n      \"endTime\": \"2026-03-18T16:30:16.704331608Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-vrcl27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vrcl27\\\" to finish creating: GET https://rp.j6925184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"name\\\": \\\"116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:29:11.853285095Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:30:16.704331608Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-vrcl27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000492900\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vrcl27\\\" to finish creating: GET https://rp.j6925184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"name\\\": \\\"116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:29:11.853285095Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:30:16.704331608Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-vrcl27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' excee...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000492920\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-vrcl27\" to finish creating: GET https://rp.j6925184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\",\n      \"name\": \"116083cc-ad76-4a20-a638-573bdc265971\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T16:29:11.853285095Z\",\n      \"endTime\": \"2026-03-18T16:30:16.704331608Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-vrcl27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vrcl27\\\" to finish creating: GET https://rp.j6925184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"name\\\": \\\"116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:29:11.853285095Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:30:16.704331608Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-vrcl27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000492900\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vrcl27\\\" to finish creating: GET https://rp.j6925184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"name\\\": \\\"116083cc-ad76-4a20-a638-573bdc265971\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:29:11.853285095Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:30:16.704331608Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-vrcl27/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' excee...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034293266127523840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T15:37:59Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"547844a4cc6c54ab665b41baad5996e1ae671b96a427951d5a1e06e8529a56f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034293266127523840","occurred_at":"2026-03-18T15:37:59Z","failed_at":"e2e","signature_id":"e525cedf7b6f56a09840d7470ec1f9ccc8f912e649e06fc5e8a670993af0f9dd","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000f4e60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-hbbshx: failed waiting for cluster=\"cluster-zstream-4-21-hbbshx\" in resourcegroup=\"rg-zstream-upgrade-4-21-nwl58q\" to finish creating: GET https://rp.j7523840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\",\n      \"name\": \"e6336c4e-2460-4b7d-90ff-a42ce9ed477c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T16:34:11.735754512Z\",\n      \"endTime\": \"2026-03-18T16:35:04.329397808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet-zstream-hbbshx' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nwl58q/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hbbshx\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-hbbshx: failed waiting for cluster=\\\"cluster-zstream-4-21-hbbshx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-nwl58q\\\" to finish creating: GET https://rp.j7523840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"name\\\": \\\"e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:34:11.735754512Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:35:04.329397808Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hbbshx' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nwl58q/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hbbshx\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0000f4e20\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-hbbshx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-nwl58q\\\" to finish creating: GET https://rp.j7523840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"name\\\": \\\"e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:34:11.735754512Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:35:04.329397808Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hbbshx' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nwl58q/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hbbshx\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000f4e60\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-hbbshx: failed waiting for cluster=\"cluster-zstream-4-21-hbbshx\" in resourcegroup=\"rg-zstream-upgrade-4-21-nwl58q\" to finish creating: GET https://rp.j7523840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\",\n      \"name\": \"e6336c4e-2460-4b7d-90ff-a42ce9ed477c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T16:34:11.735754512Z\",\n      \"endTime\": \"2026-03-18T16:35:04.329397808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet-zstream-hbbshx' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nwl58q/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hbbshx\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-hbbshx: failed waiting for cluster=\\\"cluster-zstream-4-21-hbbshx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-nwl58q\\\" to finish creating: GET https://rp.j7523840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"name\\\": \\\"e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:34:11.735754512Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:35:04.329397808Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hbbshx' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nwl58q/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hbbshx\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0000f4e20\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-hbbshx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-nwl58q\\\" to finish creating: GET https://rp.j7523840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"name\\\": \\\"e6336c4e-2460-4b7d-90ff-a42ce9ed477c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T16:34:11.735754512Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T16:35:04.329397808Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hbbshx' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nwl58q/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hbbshx\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2215914a52cc65aa1766590be950d180e57bbd410a0c1a77f33029f2d1b9057","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034293266127523840","occurred_at":"2026-03-18T15:37:59Z","failed_at":"e2e","signature_id":"d7741eaa896bf98c6a166a5f9a45fea8b49b6e7d9dcd3476a0207fdfe386ad10","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00082ca60\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00082ca60\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034292696662675456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"be491eaa2fdb0a594a4e252881eb8f29d9bfccda","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T15:35:48Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8abb28212d7198eea804b8ff5aeba37d989c9466bb6ce9bea47f6ed455ac66df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034292696662675456","occurred_at":"2026-03-18T15:35:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034291871626301440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"b6f3e3f8b34d49c7be7af42fd00d33fffa972c0b","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T15:32:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"103d2b386724fa8ecfea3d7b734cdb761b3371b6faf17b545356f218005b3235","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034291871626301440","occurred_at":"2026-03-18T15:32:32Z","failed_at":"provision","signature_id":"c97b09cb7a20aa8b7939e8e660f66d496ef90b57c9fb87926db42fa97df9df37","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-18T16:17:09.247Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-18T16:17:09.949Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:f4d4cd29-749f-4add-af25-814804090dfc cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j6301440] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j6301440.hcpsvc.osadev.cloud] image:map[digest:sha256:ee85d3b105f0865d7e9a8e12a07b194acc4b3daccabc436a5ce0811237e0ed1d registry:arohcpsvcdev.azurecr.io repository:ci-op-m0pldgmm/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-m0pldgmm/pipeline:pull workloadIdentityClientId:19499bc9-c8d8-416f-b45f-de8c45af77a8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:c9a9304b-45b5-4b61-800e-b4ab2499bbdd workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-18T16:17:09.949Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T16:17:10.215Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-18T16:17:10.461Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-18T16:17:10.461Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T16:17:10.567Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T16:17:17.274Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.23:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.23:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.20:15017, code 500: 500 Internal Server Error\"\ntime=2026-03-18T16:17:17.274Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T16:17:17.274Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T16:17:19.116Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.23:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.23:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.20:15017, code 500: 500 Internal Server Error\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:f4d4cd29-749f-4add-af25-814804090dfc cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j6301440] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j6301440.hcpsvc.osadev.cloud] image:map[digest:sha256:ee85d3b105f0865d7e9a8e12a07b194acc4b3daccabc436a5ce0811237e0ed1d registry:arohcpsvcdev.azurecr.io repository:ci-op-m0pldgmm/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-m0pldgmm/pipeline:pull workloadIdentityClientId:19499bc9-c8d8-416f-b45f-de8c45af77a8 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:c9a9304b-45b5-4b61-800e-b4ab2499bbdd workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-18T16:17:19.119Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5RU0W7aMBR9z1dc5alIJCqd1E5MfWDtpEmTJsR46rQHxzkEF8c3unZAVHz85FLajAJlb/bJucc+Pjc3sQgk8NyKhqdbKlVQQRUWF1w8QocfxpVDH8S4qk9b6Keq8QphCfeM+EbpHdz7nRARpb+gBWEsvDQl5M4q79M+pR7eG3aVCsi8biKkFj4zPhjOjKsEkbeTkKXRGGnNrQt71e+225oJW3w1rjSu2j9OswvC1kKy4iDj/zQsVAnJYKGDYfeB5j0ay+sa5/o4ys80JJiZ0XEtmAn8HHLyHUdaxq21HUtNa+0HF75rfeB68tIe95gZZ6LPDt3nnbpcCWdz3eTqqRXkmuvIfFWzrQ+Q+LLvumC7zmrlVIWd424Wx0M4EeD5yR0Reem+8+IatWHOYp5UFByzNXr9HIm1vNoP5aDA9+l0POH2wAH/bpM/X5JFW0AcAvy3+Af6ZEOrOQQUTA0fVN1QgbACHF2UKiDCF+nV5dV1dvkpG3yeDq6Hg5vh4PIh7VGe0ynOzUPa6yUbemTjaGFceWucg3TmBjt6GxfdObE/IJINNcLx69tF+9RwuSWfJ9Iny1WfanivqijJUkKoWHfMl/D6bwAAAP//ZRy9YN0EAAA=\"\ntime=2026-03-18T16:17:19.122Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T16:17:19.122Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.23:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.23:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.20:15017, code 500: 500 Internal Server Error\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.20:15017, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034291210725625856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4481,"pr_state":"closed","pr_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","final_merged_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T15:29:54Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8ae7d3346521386e3aef8a50ff2d5024d03f4e68f428fc87fabced8e4bd5d6ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034291210725625856","occurred_at":"2026-03-18T15:29:54Z","failed_at":"e2e","signature_id":"297f65c9d3fd4f59636bc8df41d91a77f5e6b0f817453239be51dcf6cc5e64cb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00013cb00\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00013cb00\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034290560461705216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4512,"pr_state":"closed","pr_sha":"ca7bbc980a35e3fc6dc73742e2b24e1d36f1178a","final_merged_sha":"ca7bbc980a35e3fc6dc73742e2b24e1d36f1178a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T15:27:19Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5ef2cc2bf0c5b507705fd8e67ffdbc54f1cb1f8edc11e9bc62de9ff64a22ca98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034290560461705216","occurred_at":"2026-03-18T15:27:19Z","failed_at":"provision","signature_id":"5eaa422d868e778161ccee29f904cb78f843436b7a782b69a0a9d54a11ac955a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations","test_suite":"step graph","failure_text":"time=2026-03-18T15:59:18.908Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-03-18T15:59:19.611Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations values=map[recycleOperatorImage:mcr.microsoft.com/aks/command/runtime@sha256:5d1fc3770795996bc22512c28adf4623ecb8a69eec3018a88e34a87bcba3366a]\ntime=2026-03-18T15:59:19.611Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations\ntime=2026-03-18T15:59:21.027Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations err=\"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp-mitigations: Unauthorized\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ef4ce0fc5b97b80d621f337890aa4d903b2b0295cdb3cf2929a4a5a18504ad","failure_pattern":"failed to apply namespace aro-hcp-mitigations: Unauthorized","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to apply namespace aro-hcp-mitigations: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f0ef4ce0fc5b97b80d621f337890aa4d903b2b0295cdb3cf2929a4a5a18504ad"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034285126040424448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"8889208b7071172a8ebe9296afe2f522a3ecc545","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T15:05:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a02a3e09b9177f48cb74c66170bb0c622fde644502af974115ddd489c79b0494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034285126040424448","occurred_at":"2026-03-18T15:05:43Z","failed_at":"e2e","signature_id":"2ced5b32a42281d957e901a6db1a38c54fa63c3d6987841b14450c7006fc7474","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0017197e0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:94]: Timed out after 600.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0017197e0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\n    {\n        s: \"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server ocm-arohcp/kube-apiserver]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f6196f274a29cf7be86ae2d44adb58a872371281d2b2032ebef876ef248d6a7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034284663660351488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4305,"pr_state":"closed","pr_sha":"f10a9bdea7427850500fd350f53d4fc0f6aab803","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T15:03:47Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"39561aa62e03cc71e48e8c1edbce2ac0bdc714c4a82ddc941dc49ad1f9750319","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034284663660351488","occurred_at":"2026-03-18T15:03:47Z","failed_at":"provision","signature_id":"59639f04409dceb8571cb3fb4a8e77c1e36ae75e64b27966fc753796dc4cf512","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-18T15:43:02.354Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-18T15:43:03.013Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:efe1ce4c-67c2-4802-b1e0-c57342ec18bb cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j0351488] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j0351488.hcpsvc.osadev.cloud] image:map[digest:sha256:4437daa84475dc7a0f0b27f08058965c2a7fddf473ee218fbcd9c222cc39de3a registry:arohcpsvcdev.azurecr.io repository:ci-op-5sqnbcdx/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-5sqnbcdx/pipeline:pull workloadIdentityClientId:33911860-31dc-4d70-a529-3868092d1fee workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:c123548d-4a78-4621-8238-b82193881ef2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-18T15:43:03.014Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T15:43:03.249Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-18T15:43:03.594Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-18T15:43:03.594Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T15:43:03.722Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T15:43:12.309Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-03-18T15:43:12.309Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T15:43:12.309Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T15:43:14.366Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:efe1ce4c-67c2-4802-b1e0-c57342ec18bb cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j0351488] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j0351488.hcpsvc.osadev.cloud] image:map[digest:sha256:4437daa84475dc7a0f0b27f08058965c2a7fddf473ee218fbcd9c222cc39de3a registry:arohcpsvcdev.azurecr.io repository:ci-op-5sqnbcdx/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-5sqnbcdx/pipeline:pull workloadIdentityClientId:33911860-31dc-4d70-a529-3868092d1fee workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:c123548d-4a78-4621-8238-b82193881ef2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-18T15:43:14.370Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6RUT2vbThC961MMOsWgFfnz+5XikoObFAqFYhyfUnpYrcbSxqsddXbWxsEfvsh2GqPasUNv4unNm515w0scCjAGimwwwC2UWrTowuEFFU9o5Jv15TAIW19lsIW+6wb/QLhAv0FCq80LPPiRAACkdy4GQZ6QwzSDNGAIlnylBdXuWzXa6wob9NIx0mxbeKjCkBcm55B7f84rUg51iazQoRFL/ojI1+l0PKEofaVjPfFXxCCjKDV6sUYfkFa8JSkdpT6i84CGUcZMC1si3zkdwl8bM20H6XlQNoglZX3F2PF2y45BqJnsvLzHmfW295qQ7wnmmknVps31c2TMDTV9Bz5bX1pfvbHT4iDjfRo9X05oPiAvrDnTnXtsHa1ejusf+Mogi511/nZ2zhhDvb3Do3aMDI+jc3vzt9G5E9N1d0RsnzdnNCZnzWrTxDla9tu8X8CT1Cc3OzKG4smFJT8/JfNYIHsUDF+6EAjJGpY1MoLYBoPopoUCZYno4aLUgh18kV5fXn9Qlzfq6uP06v/hfzfDy5vHdAB5Dm9wrq4f08EgWcMTWQ9z68tb6z3yXnSRh9fE2o+qfkYla2iZur+vD82gpXJLPk8kA0dVBg2GoKtOkrhEhmK1N3yJwfwOAAD//7ZNzUpgBQAA\"\ntime=2026-03-18T15:43:14.372Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-18T15:43:14.373Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034284199392841728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T15:02:02Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3316140f9d8890230f19bc211e36d63150505c8e49748031bd293da990fa79e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034284199392841728","occurred_at":"2026-03-18T15:02:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034282450485514240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4497,"pr_state":"closed","pr_sha":"ef00dbb4f2576014164324de3817c70002a260cc","final_merged_sha":"ef00dbb4f2576014164324de3817c70002a260cc","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T14:55:05Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e520c70bd7f9b12f735640d2086e2c0096e547c54f486952e4cf4ae6edf6670b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034282450485514240","occurred_at":"2026-03-18T14:55:05Z","failed_at":"e2e","signature_id":"0c05880dfa8a4df1558b16f9a1ca4e61123d09c644087aef770573ecc2817758","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009409a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wxwvnp\" to finish creating: GET https://rp.j5514240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\",\n      \"name\": \"70020686-e4ff-4c0e-b027-4850cd3b68c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:40:16.05219438Z\",\n      \"endTime\": \"2026-03-18T15:42:12.275796995Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wxwvnp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wxwvnp\\\" to finish creating: GET https://rp.j5514240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"name\\\": \\\"70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:16.05219438Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:12.275796995Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wxwvnp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000940960\u003e{\n            msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wxwvnp\\\" to finish creating: GET https://rp.j5514240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"name\\\": \\\"70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:16.05219438Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:12.275796995Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wxwvnp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again afte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009409a0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wxwvnp\" to finish creating: GET https://rp.j5514240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\",\n      \"name\": \"70020686-e4ff-4c0e-b027-4850cd3b68c0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:40:16.05219438Z\",\n      \"endTime\": \"2026-03-18T15:42:12.275796995Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wxwvnp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wxwvnp\\\" to finish creating: GET https://rp.j5514240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"name\\\": \\\"70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:16.05219438Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:12.275796995Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wxwvnp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000940960\u003e{\n            msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wxwvnp\\\" to finish creating: GET https://rp.j5514240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"name\\\": \\\"70020686-e4ff-4c0e-b027-4850cd3b68c0\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:16.05219438Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:12.275796995Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wxwvnp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again afte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281185084968960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"c9de0da03a321463156d873f6467f1e4750ae792","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T14:50:04Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"266549b44658daa88925fab886dfe1700fa6267f2fa2c78233ac0a806025a3e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281185084968960","occurred_at":"2026-03-18T14:50:04Z","failed_at":"e2e","signature_id":"25a3f045e3c378a27a2a130a3744fa8218ace7c7748526ec0a59926124a90dff","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc000acd110\u003e: \n    GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f2b4c9f-393b-475e-872c-13718d862016\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f2b4c9f-393b-475e-872c-13718d862016\",\n      \"name\": \"1f2b4c9f-393b-475e-872c-13718d862016\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:30:37.846077324Z\",\n      \"endTime\": \"2026-03-18T15:32:04.274089786Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet2-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h4qth6/providers/Microsoft.Network/virtualNetworks/customer-vnet2-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"MultipleErrorsOccurred\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Request-Id\": [\n                    \"469e3d3c-dab5-4186-9f1b-626c37a65092\",\n                ],\n                \"Date\": [\n                    \"Wed, 18 Mar 2026 15:39:21 GMT\",\n                ],\n                \"X-Envoy-Upstream-Service-Time\": [\"7\"],\n                \"Server\": [\"istio-envoy\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000b08400\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 49, 102, 50, 98, 52, 99, 57, 102, 45, 51, 57, 51, 98, 45, 52, 55, 53, 101, 45, 56, 55, 50, 99, 45, 49, 51, 55, 49, 56, 100, 56, 54, 50, 48, 49, 54, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 49, 102, 50, 98, 52, 99, 57, 102, 45, 51, 57, 51, 98, 45, 52, 55, 53, 101, 45, 56, 55, 50, 99, 45, 49, 51, 55, 49, 56, 100, 56, 54, 50, 48, 49, 54, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 48, 58, 51, 55, 46, 56, 52, 54, 48, 55, 55, 51, 50, 52, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 50, 58, 48, 52, 46, 50, 55, 52, 48, 56, 57, 55, 56, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 109, 117, 108, 116, 105, 112, 108, 101, 32, 101, 114, 114, 111, 114, 115, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 118, 105, 114, 116, 117, 97, 108, 32, 110, 101, 116, 119, 111, 114, 107, 32, 39, 99, 117, 115, 116, 111, 109, 101, 114, 45, 118, 110, 101, 116, 50, 45, 110, 97, 109, 101, 39, 32, 100, 101, 116, 97, 105, 108, 115, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc000acd110\u003e: \n    GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f2b4c9f-393b-475e-872c-13718d862016\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f2b4c9f-393b-475e-872c-13718d862016\",\n      \"name\": \"1f2b4c9f-393b-475e-872c-13718d862016\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:30:37.846077324Z\",\n      \"endTime\": \"2026-03-18T15:32:04.274089786Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet2-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h4qth6/providers/Microsoft.Network/virtualNetworks/customer-vnet2-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"MultipleErrorsOccurred\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Request-Id\": [\n                    \"469e3d3c-dab5-4186-9f1b-626c37a65092\",\n                ],\n                \"Date\": [\n                    \"Wed, 18 Mar 2026 15:39:21 GMT\",\n                ],\n                \"X-Envoy-Upstream-Service-Time\": [\"7\"],\n                \"Server\": [\"istio-envoy\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000b08400\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 49, 102, 50, 98, 52, 99, 57, 102, 45, 51, 57, 51, 98, 45, 52, 55, 53, 101, 45, 56, 55, 50, 99, 45, 49, 51, 55, 49, 56, 100, 56, 54, 50, 48, 49, 54, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 49, 102, 50, 98, 52, 99, 57, 102, 45, 51, 57, 51, 98, 45, 52, 55, 53, 101, 45, 56, 55, 50, 99, 45, 49, 51, 55, 49, 56, 100, 56, 54, 50, 48, 49, 54, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 48, 58, 51, 55, 46, 56, 52, 54, 48, 55, 55, 51, 50, 52, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 50, 58, 48, 52, 46, 50, 55, 52, 48, 56, 57, 55, 56, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 77, 117, 108, 116, 105, 112, 108, 101, 69, 114, 114, 111, 114, 115, 79, 99, 99, 117, 114, 114, 101, 100, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 109, 117, 108, 116, 105, 112, 108, 101, 32, 101, 114, 114, 111, 114, 115, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 100, 101, 116, 97, 105, 108, 115, 34, 58, 32, 91, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 118, 105, 114, 116, 117, 97, 108, 32, 110, 101, 116, 119, 111, 114, 107, 32, 39, 99, 117, 115, 116, 111, 109, 101, 114, 45, 118, 110, 101, 116, 50, 45, 110, 97, 109, 101, 39, 32, 100, 101, 116, 97, 105, 108, 115, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6a845dfd4cf9d39657baf7ec8f5337d64ff88820528116232320b03597690a4a","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"2d462837883fb015747af0b77a4ea1a9db7595b8bd7ce4efc534b336ef104233","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281185084968960","occurred_at":"2026-03-18T14:50:04Z","failed_at":"e2e","signature_id":"389ac41b76e352b717a9ba6e7adf0bbf9b14491dc05dec966dabb148e61dc790","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007aa280\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-b2rgrg\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\",\n      \"name\": \"0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:53:21.008761573Z\",\n      \"endTime\": \"2026-03-18T15:54:00.900097609Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b2rgrg\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"name\\\": \\\"0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:53:21.008761573Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:54:00.900097609Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007aa260\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b2rgrg\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"name\\\": \\\"0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:53:21.008761573Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:54:00.900097609Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007aa280\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-b2rgrg\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\",\n      \"name\": \"0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:53:21.008761573Z\",\n      \"endTime\": \"2026-03-18T15:54:00.900097609Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b2rgrg\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"name\\\": \\\"0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:53:21.008761573Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:54:00.900097609Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007aa260\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-b2rgrg\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"name\\\": \\\"0b0ba2cf-bb21-4b22-8217-c5b79c26b1e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:53:21.008761573Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:54:00.900097609Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-186/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n----------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"364d50bfd93543d8b018512615cbd41cbee9d6a0236eaa039945c81f55919d4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281185084968960","occurred_at":"2026-03-18T14:50:04Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"b78c61a7f55fc3a715f64fee731358a1a87b4f1caee8082ad9c0d7b61671e7fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281185084968960","occurred_at":"2026-03-18T14:50:04Z","failed_at":"e2e","signature_id":"46d872cbf68de84eae596f0a0828651c2dab45aed41d91818c7419b006ebff39","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007bea60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rm29ks\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\",\n      \"name\": \"a598f117-d124-44d5-93ef-ffae6f5ba2e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:29:40.838949406Z\",\n      \"endTime\": \"2026-03-18T15:31:03.468090691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rm29ks/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rm29ks\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"name\\\": \\\"a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:29:40.838949406Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:31:03.468090691Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rm29ks/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007bea40\u003e{\n            msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rm29ks\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"name\\\": \\\"a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:29:40.838949406Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:31:03.468090691Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rm29ks/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007bea60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rm29ks\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\",\n      \"name\": \"a598f117-d124-44d5-93ef-ffae6f5ba2e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:29:40.838949406Z\",\n      \"endTime\": \"2026-03-18T15:31:03.468090691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rm29ks/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rm29ks\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"name\\\": \\\"a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:29:40.838949406Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:31:03.468090691Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rm29ks/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007bea40\u003e{\n            msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rm29ks\\\" to finish creating: GET https://rp.j4968960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"name\\\": \\\"a598f117-d124-44d5-93ef-ffae6f5ba2e5\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:29:40.838949406Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:31:03.468090691Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rm29ks/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d6465a9678a54ab769f94d4b68a7e2b53b65cb03030e373509cb2b70ad29a39e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281185084968960","occurred_at":"2026-03-18T14:50:04Z","failed_at":"e2e","signature_id":"1e350412195edde82ef0a691c46feb155fe001e6ca2cee3cf2ab458d3e463370","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b23080\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-dcl9tx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dcl9tx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00117f460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b23080\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-dcl9tx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-dcl9tx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00117f460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-dcl9tx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-dcl9tx, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","6a845dfd4cf9d39657baf7ec8f5337d64ff88820528116232320b03597690a4a","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281018814369792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4491,"pr_state":"closed","pr_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","final_merged_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T14:49:24Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bb1ea5ce8035b1fa560fe0069624ea32e84468695173dec4f89b9d210bef002c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034281018814369792","occurred_at":"2026-03-18T14:49:24Z","failed_at":"e2e","signature_id":"cddf85bffc388bf1dec8b8035633843ef4b7756a5d79399d90b4da25bde3315c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00075b500\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-hjll2f: failed waiting for cluster=\"cluster-zstream-4-21-hjll2f\" in resourcegroup=\"rg-zstream-upgrade-4-21-v8v2zt\" to finish creating: GET https://rp.j4369792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\",\n      \"name\": \"09d881ad-ded7-4498-8e1a-ea625d67d91e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:39:54.750498141Z\",\n      \"endTime\": \"2026-03-18T15:42:18.68414891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet-zstream-hjll2f' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-v8v2zt/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hjll2f\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-hjll2f: failed waiting for cluster=\\\"cluster-zstream-4-21-hjll2f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-v8v2zt\\\" to finish creating: GET https://rp.j4369792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"name\\\": \\\"09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:54.750498141Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:18.68414891Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hjll2f' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-v8v2zt/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hjll2f\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00075b4c0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-hjll2f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-v8v2zt\\\" to finish creating: GET https://rp.j4369792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"name\\\": \\\"09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:54.750498141Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:18.68414891Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hjll2f' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-v8v2zt/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hjll2f\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a68...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00075b500\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-hjll2f: failed waiting for cluster=\"cluster-zstream-4-21-hjll2f\" in resourcegroup=\"rg-zstream-upgrade-4-21-v8v2zt\" to finish creating: GET https://rp.j4369792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\",\n      \"name\": \"09d881ad-ded7-4498-8e1a-ea625d67d91e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:39:54.750498141Z\",\n      \"endTime\": \"2026-03-18T15:42:18.68414891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Failed to get virtual network 'customer-vnet-zstream-hjll2f' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-v8v2zt/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hjll2f\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-hjll2f: failed waiting for cluster=\\\"cluster-zstream-4-21-hjll2f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-v8v2zt\\\" to finish creating: GET https://rp.j4369792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"name\\\": \\\"09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:54.750498141Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:18.68414891Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hjll2f' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-v8v2zt/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hjll2f\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00075b4c0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-hjll2f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-v8v2zt\\\" to finish creating: GET https://rp.j4369792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"name\\\": \\\"09d881ad-ded7-4498-8e1a-ea625d67d91e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:54.750498141Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:42:18.68414891Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-hjll2f' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-v8v2zt/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-hjll2f\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a68...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034280313005281280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T14:46:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278369889095680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"closed","pr_sha":"47d7e02117a2fe3ee2182231baaf80bf45181128","final_merged_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T14:38:52Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"05d61f0c2cdd83d710ea0c3ffe356dc8dc3a6cc591b25aba712c8616facf34c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278369889095680","occurred_at":"2026-03-18T14:38:52Z","failed_at":"e2e","signature_id":"fb616fbf0dfd4d6218fc989ca838d33eeae6d9bd07e1949440b1e446ec1dbf76","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0011f6d50\u003e: \n    GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3df1fad-5b33-490c-8720-40caecee25af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3df1fad-5b33-490c-8720-40caecee25af\",\n      \"name\": \"c3df1fad-5b33-490c-8720-40caecee25af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:37:10.011477634Z\",\n      \"endTime\": \"2026-03-18T15:37:57.291321656Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InternalServerError\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Request-Id\": [\n                    \"1f44e88a-3cf3-4e30-9689-672c60f0dee9\",\n                ],\n                \"Date\": [\n                    \"Wed, 18 Mar 2026 15:38:00 GMT\",\n                ],\n                \"Content-Length\": [\"1561\"],\n                \"X-Envoy-Upstream-Service-Time\": [\"6\"],\n                \"Server\": [\"istio-envoy\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc0004c6c60\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 99, 51, 100, 102, 49, 102, 97, 100, 45, 53, 98, 51, 51, 45, 52, 57, 48, 99, 45, 56, 55, 50, 48, 45, 52, 48, 99, 97, 101, 99, 101, 101, 50, 53, 97, 102, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 99, 51, 100, 102, 49, 102, 97, 100, 45, 53, 98, 51, 51, 45, 52, 57, 48, 99, 45, 56, 55, 50, 48, 45, 52, 48, 99, 97, 101, 99, 101, 101, 50, 53, 97, 102, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 55, 58, 49, 48, 46, 48, 49, 49, 52, 55, 55, 54, 51, 52, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 55, 58, 53, 55, 46, 50, 57, 49, 51, 50, 49, 54, 53, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 50, 50, 51, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0011f6d50\u003e: \n    GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3df1fad-5b33-490c-8720-40caecee25af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3df1fad-5b33-490c-8720-40caecee25af\",\n      \"name\": \"c3df1fad-5b33-490c-8720-40caecee25af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:37:10.011477634Z\",\n      \"endTime\": \"2026-03-18T15:37:57.291321656Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-223/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InternalServerError\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Request-Id\": [\n                    \"1f44e88a-3cf3-4e30-9689-672c60f0dee9\",\n                ],\n                \"Date\": [\n                    \"Wed, 18 Mar 2026 15:38:00 GMT\",\n                ],\n                \"Content-Length\": [\"1561\"],\n                \"X-Envoy-Upstream-Service-Time\": [\"6\"],\n                \"Server\": [\"istio-envoy\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc0004c6c60\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 99, 51, 100, 102, 49, 102, 97, 100, 45, 53, 98, 51, 51, 45, 52, 57, 48, 99, 45, 56, 55, 50, 48, 45, 52, 48, 99, 97, 101, 99, 101, 101, 50, 53, 97, 102, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 99, 51, 100, 102, 49, 102, 97, 100, 45, 53, 98, 51, 51, 45, 52, 57, 48, 99, 45, 56, 55, 50, 48, 45, 52, 48, 99, 97, 101, 99, 101, 101, 50, 53, 97, 102, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 55, 58, 49, 48, 46, 48, 49, 49, 52, 55, 55, 54, 51, 52, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 51, 55, 58, 53, 55, 46, 50, 57, 49, 51, 50, 49, 54, 53, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 50, 50, 51, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 111, 110, 116, 114, 111, 108, 45, 112, 108, 97, 110, 101, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"73a8d5cb21bf029cd1494ec2d80a9707ff57e9fc00d3ae3b35698600e6cdf456","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278369889095680","occurred_at":"2026-03-18T14:38:52Z","failed_at":"e2e","signature_id":"1e00c622004b27a3656ff98432e5eca1d3ae6593c0bee38b668b029135d3c3fa","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0015e40e0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\": dial tcp 51.57.113.19:443: connect: connection timed out\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\\\": dial tcp 51.57.113.19:443: connect: connection timed out\",\n        err: \u003c*url.Error | 0xc000a48540\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\",\n            Err: \u003c*net.OpError | 0xc000d9e000\u003e{\n                Op: \"dial\",\n                Net: \"tcp\",\n                Source: nil,\n                Addr: \u003c*net.TCPAddr | 0xc000a489c0\u003e{\n                    IP: [51, 57, 113, 19],\n                    Port: 443,\n                    Zone: \"\",\n                },\n                Err: \u003c*os.SyscallError | 0xc0015e40a0\u003e{\n                    Syscall: \"connect\",\n                    Err: \u003csyscall.Errno\u003e0x6e,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:137]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0015e40e0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\": dial tcp 51.57.113.19:443: connect: connection timed out\n    {\n        msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\\\": dial tcp 51.57.113.19:443: connect: connection timed out\",\n        err: \u003c*url.Error | 0xc000a48540\u003e{\n            Op: \"Get\",\n            URL: \"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\",\n            Err: \u003c*net.OpError | 0xc000d9e000\u003e{\n                Op: \"dial\",\n                Net: \"tcp\",\n                Source: nil,\n                Addr: \u003c*net.TCPAddr | 0xc000a489c0\u003e{\n                    IP: [51, 57, 113, 19],\n                    Port: 443,\n                    Zone: \"\",\n                },\n                Err: \u003c*os.SyscallError | 0xc0015e40a0\u003e{\n                    Syscall: \"connect\",\n                    Err: \u003csyscall.Errno\u003e0x6e,\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: connect: connection timed out","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-p8q8d.apps.aro.z3v9u1z0j7n5w0w.fdfh.j9095680.hcp.osadev.cloud\": dial tcp 51.57.113.19:443: connect: connection timed out"}},{"row_id":"9a0de6a4add4630a3b6c7c1dc07a634a4ab0ed550aed2135be186338430c6ef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278369889095680","occurred_at":"2026-03-18T14:38:52Z","failed_at":"e2e","signature_id":"42da6a4e2e8bb20de7f4cb376bb13131244c3bb25a152f8e60ecdcf41016cf36","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008e1000\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-dlct2d\" to finish creating: GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\",\n      \"name\": \"6d3ea23e-abd5-4169-a804-e84509b8668d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:37:05.133933797Z\",\n      \"endTime\": \"2026-03-18T15:38:56.831751444Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-dlct2d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlct2d\\\" to finish creating: GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"name\\\": \\\"6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:37:05.133933797Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:38:56.831751444Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-dlct2d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008e0fe0\u003e{\n            msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlct2d\\\" to finish creating: GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"name\\\": \\\"6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:37:05.133933797Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:38:56.831751444Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-dlct2d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again af...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008e1000\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-dlct2d\" to finish creating: GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\",\n      \"name\": \"6d3ea23e-abd5-4169-a804-e84509b8668d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:37:05.133933797Z\",\n      \"endTime\": \"2026-03-18T15:38:56.831751444Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-dlct2d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlct2d\\\" to finish creating: GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"name\\\": \\\"6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:37:05.133933797Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:38:56.831751444Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-dlct2d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008e0fe0\u003e{\n            msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dlct2d\\\" to finish creating: GET https://rp.j9095680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"name\\\": \\\"6d3ea23e-abd5-4169-a804-e84509b8668d\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:37:05.133933797Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:38:56.831751444Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-dlct2d/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again af...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2756b831e4b1e0a56a7949bf61fab69ac22501180f2bc329512550fb97dec7fa","503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","5425136962af76243adc80d6eb9654ea6e21ebe16918744ee896a67881c617b8"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4508/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278233309974528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4508,"pr_state":"closed","pr_sha":"a2ca195603032cc033a8e2ebe803918b756d5580","final_merged_sha":"4da0d9f5994ef9590af727a3ca8f65f2ef8646b8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T14:38:20Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f0da24c7d15f50feb390838d99c652322fd4ae6f5d6d698790c9b0bbbacf173d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4508/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278233309974528","occurred_at":"2026-03-18T14:38:20Z","failed_at":"provision","signature_id":"f9d01b04ac3f70b01e1c221dc5c2be37577429e2ba7febe11e7da750ebf0feac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.HypershiftOperator/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-18T15:15:08.938Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e hypershift.image.registry \u003cnil\u003e}:{\u003cnil\u003e hypershift.image.repository \u003cnil\u003e}@{\u003cnil\u003e hypershift.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-18T15:15:13.751Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image quay.io/redhat-services-prod/crt-redhat-acm-tenant/hypershift/hypershift-operator@sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7 to arohcpsvcdev.azurecr.io/redhat-services-prod/crt-redhat-acm-tenant/hypershift/hypershift-operator:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7.\\nThe image will still be available under it's original digest sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7 in the target registry.\\nError: failed to resolve sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7: GET \\\"https://quay.io/v2/redhat-services-prod/crt-redhat-acm-tenant/hypershift/hypershift-operator/manifests/sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7\\\": invalid response; digest mismatch in Docker-Content-Digest: received \\\"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\\\" when expecting \\\"sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7\\\"\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"78a114073bd8d8fefe6a2203dd05d95d9ad6af0d1d9a2bb1e09c5ae28d3362ff","failure_pattern":"Error: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": invalid response; digest mismatch in Docker-Content-Digest: received \"sha256:\u003chex\u003e\" when expecting \"sha256:\u003chex\u003e\"","search_query":"Error: failed to resolve sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7: GET \\\"https://quay.io/v2/redhat-services-prod/crt-redhat-acm-tenant/hypershift/hypershift-operator/manifests/sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7\\\": invalid response; digest mismatch in Docker-Content-Digest: received \\\"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\\\" when expecting \\\"sha256:7f027c55284ca274ff434b2866196b8392d0785bb97dc32e8d4abc18909fbdf7\\\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["78a114073bd8d8fefe6a2203dd05d95d9ad6af0d1d9a2bb1e09c5ae28d3362ff"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278102632239104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4347,"pr_state":"closed","pr_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","final_merged_sha":"a82c62541bf569a06efb0a6211c6ae9e8a2c01a4","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T14:37:49Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"19de2c9b57825a0f402c000ddfac1a411b1256ae739fff3fe460e8aeab8c8f4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4347/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034278102632239104","occurred_at":"2026-03-18T14:37:49Z","failed_at":"e2e","signature_id":"edfac7b4ab4f1d1c77f7ee1ca2e3fcb0f4a6e05a8d2d82080056d41d6d1a98a1","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0006754c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-wjcpfb: failed waiting for cluster=\"cluster-zstream-4-19-wjcpfb\" in resourcegroup=\"rg-zstream-upgrade-4-19-cct4x4\" to finish creating: GET https://rp.j2239104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\",\n      \"name\": \"161f28e8-f27b-44a6-985c-acbfac3cb120\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:48:29.77050876Z\",\n      \"endTime\": \"2026-03-18T15:49:46.080405554Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-zstream-wjcpfb' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cct4x4/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-wjcpfb\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-wjcpfb: failed waiting for cluster=\\\"cluster-zstream-4-19-wjcpfb\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cct4x4\\\" to finish creating: GET https://rp.j2239104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\\\",\\n  \\\"name\\\": \\\"161f28e8-f27b-44a6-985c-acbfac3cb120\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:48:29.77050876Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:49:46.080405554Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-wjcpfb' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cct4x4/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-wjcpfb\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0006754a0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-wjcpfb\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cct4x4\\\" to finish creating: GET https://rp.j2239104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfa...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0006754c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-wjcpfb: failed waiting for cluster=\"cluster-zstream-4-19-wjcpfb\" in resourcegroup=\"rg-zstream-upgrade-4-19-cct4x4\" to finish creating: GET https://rp.j2239104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\",\n      \"name\": \"161f28e8-f27b-44a6-985c-acbfac3cb120\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:48:29.77050876Z\",\n      \"endTime\": \"2026-03-18T15:49:46.080405554Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-zstream-wjcpfb' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cct4x4/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-wjcpfb\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-wjcpfb: failed waiting for cluster=\\\"cluster-zstream-4-19-wjcpfb\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cct4x4\\\" to finish creating: GET https://rp.j2239104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfac3cb120\\\",\\n  \\\"name\\\": \\\"161f28e8-f27b-44a6-985c-acbfac3cb120\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:48:29.77050876Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:49:46.080405554Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-243/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-wjcpfb' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cct4x4/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-wjcpfb\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0006754a0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-wjcpfb\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cct4x4\\\" to finish creating: GET https://rp.j2239104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/161f28e8-f27b-44a6-985c-acbfa...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0aa9ed5d94e7167a12aa30282eff19539fc3fc473772cb96b223d420df66775f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034275672028876800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4461,"pr_state":"closed","pr_sha":"9baa329552156f993c1fc5b382b78423cd754fef","final_merged_sha":"9baa329552156f993c1fc5b382b78423cd754fef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T14:28:09Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"05d0b6eb0d06c7d0dc9157f84f8f2b58c09019a7b49bb474fd1df56f748eaf52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034275672028876800","occurred_at":"2026-03-18T14:28:09Z","failed_at":"e2e","signature_id":"90063775c1017d4fd9c68eaeaf236277d301df7fc213b3484ab0528af4aafff1","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00136e440\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sbghmb\" to finish creating: GET https://rp.j8876800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\",\n      \"name\": \"7163cf65-26e8-4c55-8654-25f332c3301c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:39:30.642744115Z\",\n      \"endTime\": \"2026-03-18T15:40:26.468431346Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sbghmb\\\" to finish creating: GET https://rp.j8876800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"name\\\": \\\"7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:30.642744115Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:40:26.468431346Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00136e420\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sbghmb\\\" to finish creating: GET https://rp.j8876800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"name\\\": \\\"7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:30.642744115Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:40:26.468431346Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00136e440\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sbghmb\" to finish creating: GET https://rp.j8876800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\",\n      \"name\": \"7163cf65-26e8-4c55-8654-25f332c3301c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:39:30.642744115Z\",\n      \"endTime\": \"2026-03-18T15:40:26.468431346Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sbghmb\\\" to finish creating: GET https://rp.j8876800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"name\\\": \\\"7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:30.642744115Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:40:26.468431346Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00136e420\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-sbghmb\\\" to finish creating: GET https://rp.j8876800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"name\\\": \\\"7163cf65-26e8-4c55-8654-25f332c3301c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:39:30.642744115Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:40:26.468431346Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-151/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n-------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034275036151418880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4496,"pr_state":"open","pr_sha":"495553567b5b241eec5af7679c41f00549f35597","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T14:25:38Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3bffdf1c2c010aa35aa27c83de84e3af4ea844498fa279d647f79f01331ec5c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034275036151418880","occurred_at":"2026-03-18T14:25:38Z","failed_at":"e2e","signature_id":"e9ca0356da0daf3a4a8d32ac491accada58a52f71125d68818f10b267f0c47dd","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0006c71d0\u003e: \n    GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6784506-194c-4bbc-ad90-7a9253307720\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6784506-194c-4bbc-ad90-7a9253307720\",\n      \"name\": \"c6784506-194c-4bbc-ad90-7a9253307720\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:42:08.505361166Z\",\n      \"endTime\": \"2026-03-18T15:43:16.011043158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InternalServerError\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Length\": [\"1581\"],\n                \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                \"Server\": [\"istio-envoy\"],\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Request-Id\": [\n                    \"7b736841-ba39-45f8-96c3-140caef5a721\",\n                ],\n                \"Date\": [\n                    \"Wed, 18 Mar 2026 15:50:42 GMT\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000aa2660\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 99, 54, 55, 56, 52, 53, 48, 54, 45, 49, 57, 52, 99, 45, 52, 98, 98, 99, 45, 97, 100, 57, 48, 45, 55, 97, 57, 50, 53, 51, 51, 48, 55, 55, 50, 48, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 99, 54, 55, 56, 52, 53, 48, 54, 45, 49, 57, 52, 99, 45, 52, 98, 98, 99, 45, 97, 100, 57, 48, 45, 55, 97, 57, 50, 53, 51, 51, 48, 55, 55, 50, 48, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 52, 50, 58, 48, 56, 46, 53, 48, 53, 51, 54, 49, 49, 54, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 52, 51, 58, 49, 54, 46, 48, 49, 49, 48, 52, 51, 49, 53, 56, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 54, 55, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 108, 111, 117, 100, 45, 99, 111, 110, 116, 114, 111, 108, 108, 101, 114, 45, 109, 97, 110, 97, 103, 101, 114, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0006c71d0\u003e: \n    GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6784506-194c-4bbc-ad90-7a9253307720\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6784506-194c-4bbc-ad90-7a9253307720\",\n      \"name\": \"c6784506-194c-4bbc-ad90-7a9253307720\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:42:08.505361166Z\",\n      \"endTime\": \"2026-03-18T15:43:16.011043158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InternalServerError\",\n        StatusCode: 200,\n        RawResponse: {\n            Status: \"200 OK\",\n            StatusCode: 200,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Length\": [\"1581\"],\n                \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                \"Server\": [\"istio-envoy\"],\n                \"Content-Type\": [\"application/json\"],\n                \"X-Ms-Request-Id\": [\n                    \"7b736841-ba39-45f8-96c3-140caef5a721\",\n                ],\n                \"Date\": [\n                    \"Wed, 18 Mar 2026 15:50:42 GMT\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000aa2660\u003e{\n                s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101, 115, 47, 99, 54, 55, 56, 52, 53, 48, 54, 45, 49, 57, 52, 99, 45, 52, 98, 98, 99, 45, 97, 100, 57, 48, 45, 55, 97, 57, 50, 53, 51, 51, 48, 55, 55, 50, 48, 34, 44, 10, 32, 32, 32, 32, 34, 110, 97, 109, 101, 34, 58, 32, 34, 99, 54, 55, 56, 52, 53, 48, 54, 45, 49, 57, 52, 99, 45, 52, 98, 98, 99, 45, 97, 100, 57, 48, 45, 55, 97, 57, 50, 53, 51, 51, 48, 55, 55, 50, 48, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 116, 117, 115, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 34, 44, 10, 32, 32, 32, 32, 34, 115, 116, 97, 114, 116, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 52, 50, 58, 48, 56, 46, 53, 48, 53, 51, 54, 49, 49, 54, 54, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 110, 100, 84, 105, 109, 101, 34, 58, 32, 34, 50, 48, 50, 54, 45, 48, 51, 45, 49, 56, 84, 49, 53, 58, 52, 51, 58, 49, 54, 46, 48, 49, 49, 48, 52, 51, 49, 53, 56, 90, 34, 44, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 102, 97, 105, 108, 101, 100, 32, 116, 111, 32, 103, 101, 116, 32, 109, 97, 110, 97, 103, 101, 100, 32, 105, 100, 101, 110, 116, 105, 116, 121, 32, 39, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 114, 101, 115, 111, 117, 114, 99, 101, 71, 114, 111, 117, 112, 115, 47, 97, 114, 111, 45, 104, 99, 112, 45, 116, 101, 115, 116, 45, 109, 115, 105, 45, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 45, 100, 101, 118, 45, 54, 55, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 77, 97, 110, 97, 103, 101, 100, 73, 100, 101, 110, 116, 105, 116, 121, 47, 117, 115, 101, 114, 65, 115, 115, 105, 103, 110, 101, 100, 73, 100, 101, 110, 116, 105, 116, 105, 101, 115, 47, 99, 108, 111, 117, 100, 45, 99, 111, 110, 116, 114, 111, 108, 108, 101, 114, 45, 109, 97, 110, 97, 103, 101, 114, 39, 58, 32, 71, 69, 84, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 122, 117, 114, 101, 46, 99, 111, 109, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100,...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8468a3e484599d30bb38736e9a60791ce3419bf2c83088e84451dba98b54edcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034275036151418880","occurred_at":"2026-03-18T14:25:38Z","failed_at":"e2e","signature_id":"e3f954074448075a9845ec3b52841b0e2f784245c6a7a702b1fcade63c5dcb5e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b046a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-7bt5td: failed waiting for cluster=\"cluster-zstream-4-19-7bt5td\" in resourcegroup=\"rg-zstream-upgrade-4-19-b9prxw\" to finish creating: GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\",\n      \"name\": \"96c91453-1ae7-4e80-992f-9ac2bf0c1af8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:40:25.336190273Z\",\n      \"endTime\": \"2026-03-18T15:41:06.719672779Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-7bt5td: failed waiting for cluster=\\\"cluster-zstream-4-19-7bt5td\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-b9prxw\\\" to finish creating: GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"name\\\": \\\"96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:25.336190273Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:41:06.719672779Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b04680\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-7bt5td\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-b9prxw\\\" to finish creating: GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"name\\\": \\\"96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:25.336190273Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:41:06.719672779Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b046a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-7bt5td: failed waiting for cluster=\"cluster-zstream-4-19-7bt5td\" in resourcegroup=\"rg-zstream-upgrade-4-19-b9prxw\" to finish creating: GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\",\n      \"name\": \"96c91453-1ae7-4e80-992f-9ac2bf0c1af8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T15:40:25.336190273Z\",\n      \"endTime\": \"2026-03-18T15:41:06.719672779Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-7bt5td: failed waiting for cluster=\\\"cluster-zstream-4-19-7bt5td\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-b9prxw\\\" to finish creating: GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"name\\\": \\\"96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:25.336190273Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:41:06.719672779Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b04680\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-7bt5td\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-b9prxw\\\" to finish creating: GET https://rp.j1418880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"name\\\": \\\"96c91453-1ae7-4e80-992f-9ac2bf0c1af8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T15:40:25.336190273Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T15:41:06.719672779Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034259246979223552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4481,"pr_state":"closed","pr_sha":"7b93e84ac1e28a465ef5f1b9405578fb8b024d89","final_merged_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T13:22:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034247968894226432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T12:37:55Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7c74f5ce03c8c7c2e10102c22bd3563334dd1d67cb8c376c8727b3e3620f4ffa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034247968894226432","occurred_at":"2026-03-18T12:37:55Z","failed_at":"e2e","signature_id":"bc359308840047e5a4d944e1fc9c5910f6393c747587889b6f8b496f35ae697c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000d40360\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-nwdhx9\" to finish creating: GET https://rp.j4226432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\",\n      \"name\": \"4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T13:27:52.504509033Z\",\n      \"endTime\": \"2026-03-18T13:29:26.947856695Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-nwdhx9\\\" to finish creating: GET https://rp.j4226432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"name\\\": \\\"4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T13:27:52.504509033Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T13:29:26.947856695Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000d40340\u003e{\n            msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-nwdhx9\\\" to finish creating: GET https://rp.j4226432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"name\\\": \\\"4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T13:27:52.504509033Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T13:29:26.947856695Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000d40360\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-nwdhx9\" to finish creating: GET https://rp.j4226432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\",\n      \"name\": \"4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T13:27:52.504509033Z\",\n      \"endTime\": \"2026-03-18T13:29:26.947856695Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-nwdhx9\\\" to finish creating: GET https://rp.j4226432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"name\\\": \\\"4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T13:27:52.504509033Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T13:29:26.947856695Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"SubscriptionRequestsThrottled\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'd6b62dfa-87f5-49b3-bbcb-4a687c4faa96' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\\\\\"\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000d40340\u003e{\n            msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-nwdhx9\\\" to finish creating: GET https://rp.j4226432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"name\\\": \\\"4e2f64df-6d2a-4cdf-bddb-1c2e60f9a6f3\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T13:27:52.504509033Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T13:29:26.947856695Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager': GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-dev-91/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 429: 429 Too Many Requests\\\\nERROR CODE: SubscriptionRequestsThrottled\\\\n---------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["503df90ff2dcdbf26a790afabd969fd677d3c7f1ed33ad2f6da0a4a32865efe7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034247248031780864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"476f04da92b1b405fb173cc4435d154017184bcc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T12:35:13Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"adce6dad89f94657c786a16ad0abfb1e8ae3d51706501478df9b99003c7760bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034247248031780864","occurred_at":"2026-03-18T12:35:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034245836992417792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4491,"pr_state":"closed","pr_sha":"30983c79692899ed6ef6a35655e4cc1b994320ef","final_merged_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T12:29:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034245208580820992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4491,"pr_state":"closed","pr_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","final_merged_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T12:27:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034245047876063232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4491,"pr_state":"closed","pr_sha":"30983c79692899ed6ef6a35655e4cc1b994320ef","final_merged_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T12:26:28Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"077c26da4ed06ed3ec58197715bf4dc60c034b73e8bddcbbab837f482cff597e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034245047876063232","occurred_at":"2026-03-18T12:26:28Z","failed_at":"e2e","signature_id":"9cb135814059270371a6d90b3cafc0d090318df3706d8244f6fcdd7950db0fe7","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e023d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2v58jh\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000eb1140\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-2v58jh\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000eb1120\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-2v58jh\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000eb1100\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-2v58jh\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e023d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2v58jh\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000eb1140\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-2v58jh\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000eb1120\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-2v58jh\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000eb1100\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-2v58jh\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"30d2de3090438a2417be3f276e128eb4a9417f5478edf9f3806d97d95124f76d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034245047876063232","occurred_at":"2026-03-18T12:26:28Z","failed_at":"e2e","signature_id":"8d5d702b8bb686d23af49f86bca78c6c1cb4ca5aa0997e4b21ded616bcb737aa","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001584240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list4k2n65\" in resourcegroup=\"ea-list-vdf4nx\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0004810c0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list4k2n65\\\" in resourcegroup=\\\"ea-list-vdf4nx\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0004810a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list4k2n65\\\" in resourcegroup=\\\"ea-list-vdf4nx\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000481080\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-list4k2n65\\\" in resourcegroup=\\\"ea-list-vdf4nx\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001584240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list4k2n65\" in resourcegroup=\"ea-list-vdf4nx\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0004810c0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list4k2n65\\\" in resourcegroup=\\\"ea-list-vdf4nx\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0004810a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-list4k2n65\\\" in resourcegroup=\\\"ea-list-vdf4nx\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000481080\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-list4k2n65\\\" in resourcegroup=\\\"ea-list-vdf4nx\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}},{"row_id":"4d5effb9b62a56058e46bb6af152b2dd44a4e44c1484e133b2bbe0d1be8726ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034245047876063232","occurred_at":"2026-03-18T12:26:28Z","failed_at":"e2e","signature_id":"1fc8c97b245495d4a5bc83299f56128285d34cacd615799775dcbb8318cdd996","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c82c30\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-5x6tvh\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0009494c0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-5x6tvh\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0009494a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-5x6tvh\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000949480\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-5x6tvh\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c82c30\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-5x6tvh\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0009494c0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-5x6tvh\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0009494a0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-5x6tvh\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000949480\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"customer-rg-5x6tvh\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb09b0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-tbxqkj\" in resourcegroup=\"admin-credential-lifecycle-test-j5ckd5\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4192/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034244502398439424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4192,"pr_state":"open","pr_sha":"898324bd1a05d1d2724b813f023773015d309c79","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T12:24:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e4aa5c66874304597f02522719b6dd67896039d93ac864366772810cdcf613d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4192/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034244502398439424","occurred_at":"2026-03-18T12:24:18Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-03-18T15:11:59Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18cbda64eadcb87cbc10a63fe854437322f9599462a64da0d8bea26fcec9782c","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["18cbda64eadcb87cbc10a63fe854437322f9599462a64da0d8bea26fcec9782c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034244268750540800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4193,"pr_state":"open","pr_sha":"5caac0ba1e9e9fe7dfff1f27fcb3550065f4e1d8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T12:23:22Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"257b13a1b177e29238013ddff10591dc2381b14fb6dd9589887df2e1f9896ab9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034244268750540800","occurred_at":"2026-03-18T12:23:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034236722681745408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4193,"pr_state":"open","pr_sha":"5fab4cf64f543fe927c6e376a0ce89dbab244ecf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T11:53:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7b5b23580079aab4a0175ff4bb84d7018bc80458e66fbdf72354a77858fa9958","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034236722681745408","occurred_at":"2026-03-18T11:53:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034234992279687168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4494,"pr_state":"closed","pr_sha":"b3b9232277fc6a73440c88df8c94cc66dc7cb7ca","final_merged_sha":"b6d9c642ca814d40f9bb7032fbbf894bc4edfd69","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T11:46:29Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"10a1a6ae213e501ed6b8efaf903e5e979cfc91dcb59dabcb8408d5d8c6f73bbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034234992279687168","occurred_at":"2026-03-18T11:46:29Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034229300156174336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"b6f3e3f8b34d49c7be7af42fd00d33fffa972c0b","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T11:23:53Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"96e0dd4282cf2f5582e6f7294100f7189bcd2066063973a80531a1e17dacb66c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034229300156174336","occurred_at":"2026-03-18T11:23:53Z","failed_at":"provision","signature_id":"5a846abd85dc2ae0153625a4ece6dd86b1479b415383fc7ac860df17c29e8b47","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-18T12:03:36.219Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e backend.image.registry \u003cnil\u003e}:{\u003cnil\u003e backend.image.repository \u003cnil\u003e}@{\u003cnil\u003e backend.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-18T12:03:48.613Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build02.ci.openshift.org\\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.3jYMm47kT5/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n    sock = connection.create_connection(\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n    response = self._make_request(\\n               ^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n    raise new_e\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n    self._validate_conn(conn)\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n    conn.connect()\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n    self.sock = sock = self._new_conn()\\n                       ^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n    raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n    resp = conn.urlopen(\\n           ^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n    retries = retries.increment(\\n              ^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\\n    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n    cmd_result = self.invocation.execute(args)\\n                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n    raise ex\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n    results.append(self._run_job(expanded_arg, cmd_copy))\\n                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n    result = cmd_copy(params)\\n             ^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n    return self.handler(*args, **kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n    return op(**command_args)\\n           ^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n    login_server, _, password = get_login_credentials(\\n                                ^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n    return _get_credentials(cmd,\\n           ^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n    return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n                                     ^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 285, in _get_aad_token\\n    return _get_aad_token_after_challenge(cli_ctx,\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 159, in _get_aad_token_after_challenge\\n    response = requests.post(url=authhost, data=urlencode(content), headers=headers,\\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 115, in post\\n    return request(\\\"post\\\", url, data=data, json=json, **kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n    return session.request(method=method, url=url, **kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n    resp = self.send(prep, **send_kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n    r = adapter.send(request, **kwargs)\\n        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n    raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /oauth2/exchange (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9a2a4cf3344f5eab04c519ebc438e0419dbf08c2ef1692b2ed6fd7228e53ded3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034223764606029824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4351,"pr_state":"closed","pr_sha":"131806c6037c274a80c20df0a02233247a78c57e","final_merged_sha":"131806c6037c274a80c20df0a02233247a78c57e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T11:01:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034218280821985280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4504,"pr_state":"open","pr_sha":"8597c397d60104e34a3ce54cba5c543aa5c00ac7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T10:40:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034216639158816768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"a088e357d8938deb4bb0e42685c52c9ee6a7baa9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T10:33:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034213933497192448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4503,"pr_state":"closed","pr_sha":"16b44df65e798f2ce166f58b4b3cdc45fae3ccec","final_merged_sha":"3a9b5c9cb4bcc81a28d2a98b1dd50a3c686f225a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T10:22:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034212752737701888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T10:18:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034209436016316416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"02baee7afcf57dd606ee9b602da9d943ecfb8697","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T10:04:57Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9068223cb642dab06a9db8e8c7f40204b864f5c691ba3afc395b16b18fd26840","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034209436016316416","occurred_at":"2026-03-18T10:04:57Z","failed_at":"provision","signature_id":"9df96cedd304ce960ec46d168dc0a0b5c62065ee44735bac356e14225c792ada","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-18T10:38:19.959Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-18T10:38:33.511Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 204, in _new_conn\\n    sock = connection.create_connection(\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\\\", line 60, in create_connection\\n    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/python3.12/socket.py\\\", line 963, in getaddrinfo\\n    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nsocket.gaierror: [Errno -2] Name or service not known\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 787, in urlopen\\n    response = self._make_request(\\n               ^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 488, in _make_request\\n    raise new_e\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 464, in _make_request\\n    self._validate_conn(conn)\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 1093, in _validate_conn\\n    conn.connect()\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 759, in connect\\n    self.sock = sock = self._new_conn()\\n                       ^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\\\", line 211, in _new_conn\\n    raise NameResolutionError(self.host, self, e) from e\\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\n\\nThe above exception was the direct cause of the following exception:\\n\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 667, in send\\n    resp = conn.urlopen(\\n           ^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\\\", line 841, in urlopen\\n    retries = retries.increment(\\n              ^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\\\", line 535, in increment\\n    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\\n    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\n\\nDuring handling of the above exception, another exception occurred:\\n\\nTraceback (most recent call last):\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\\\", line 233, in invoke\\n    cmd_result = self.invocation.execute(args)\\n                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 682, in execute\\n    raise ex\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 812, in _run_jobs_serially\\n    results.append(self._run_job(expanded_arg, cmd_copy))\\n                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 781, in _run_job\\n    result = cmd_copy(params)\\n             ^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\\\", line 336, in __call__\\n    return self.handler(*args, **kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\\\", line 120, in handler\\n    return op(**command_args)\\n           ^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\\\", line 294, in acr_login\\n    login_server, _, password = get_login_credentials(\\n                                ^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 505, in get_login_credentials\\n    return _get_credentials(cmd,\\n           ^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 449, in _get_credentials\\n    return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\\n                                     ^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 276, in _get_aad_token\\n    token_params = _handle_challenge_phase(\\n                   ^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\\\", line 100, in _handle_challenge_phase\\n    challenge = requests.get(request_url, verify=not should_disable_connection_verify())\\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 73, in get\\n    return request(\\\"get\\\", url, params=params, **kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\\\", line 59, in request\\n    return session.request(method=method, url=url, **kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 589, in request\\n    resp = self.send(prep, **send_kwargs)\\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\\\", line 703, in send\\n    r = adapter.send(request, **kwargs)\\n        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\n  File \\\"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\\\", line 700, in send\\n    raise ConnectionError(e, request=request)\\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))\\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3","failure_pattern":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve","search_query":"requests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\\\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\\\"))"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ee5b1a1a3315ac34a62d3e4a95293e1f5e2f2b1ef572b3ff8359f4b3944b63b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034187844494299136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4502,"pr_state":"closed","pr_sha":"3d35775c67142728f2988011679a10fafb63f20a","final_merged_sha":"3d35775c67142728f2988011679a10fafb63f20a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T08:39:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034185206918811648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4491,"pr_state":"closed","pr_sha":"30983c79692899ed6ef6a35655e4cc1b994320ef","final_merged_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T08:28:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034181135944126464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4481,"pr_state":"closed","pr_sha":"299f5e8421000685157cbc27db66b8219a2d8d7b","final_merged_sha":"a0ece52bde4e4930997796ae17d728b08f758b6b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T08:12:30Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3e2c875657eda6a62ea94c38ee805c970de6947f89fbbd8f24bb57a3af011006","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4481/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034181135944126464","occurred_at":"2026-03-18T08:12:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034178947427602432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4224,"pr_state":"closed","pr_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","final_merged_sha":"0d2c310eee81cd65f33fd842930856c19dc160e3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T08:03:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034156129121996800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4403,"pr_state":"open","pr_sha":"d44789f94e28ef6703ad67f22b74ac511f48ebdc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T06:33:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034154020188196864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4491,"pr_state":"closed","pr_sha":"30983c79692899ed6ef6a35655e4cc1b994320ef","final_merged_sha":"5a75b4ff9e4b4c37dc89c577e197e6efe95da36f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-18T06:24:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034117300587597824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4219,"pr_state":"closed","pr_sha":"c818402f9cb5d03c3c1e1304e7b9ee68b672c2e0","final_merged_sha":"8550ece78eb685f5f0357a6937bf6b6e9f457543","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-18T03:58:51Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6dc709d2ba3699a4e3abe2f00ef4bca53e457cc6cea3e7d6b0aa0cc4684449a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034117300587597824","occurred_at":"2026-03-18T03:58:51Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034093855069966336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T02:25:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2034084363141910528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4498,"pr_state":"closed","pr_sha":"30c99a54600726981d1afad182e57eb6ff611d65","final_merged_sha":"30c99a54600726981d1afad182e57eb6ff611d65","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T01:47:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}