{"meta":{"date":"2026-04-02","timezone":"UTC","generated_at":"2026-08-30T11:18:22Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":11,"runs_with_occurrences":11,"runs_with_failure_pattern_matches":11,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T22:49:32Z"},"failed_at":["e2e","unknown"],"failed_test_count":15,"occurrences":[{"row_id":"0bafc51fe4cf17be3abdf8374a1896eb79b602295fc73504f87b8589fc16dd01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"77bc18f4976eeb2ac0f4331febecd092cfe9f711cc268f53af542f3b3632ccb2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008c0420\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-jrgxnr\" in resourcegroup=\"rg-zstream-upgrade-4-19-5mbcpg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-jrgxnr in resource group rg-zstream-upgrade-4-19-5mbcpg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-jrgxnr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-5mbcpg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-jrgxnr in resource group rg-zstream-upgrade-4-19-5mbcpg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0007fd0b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-jrgxnr in resource group rg-zstream-upgrade-4-19-5mbcpg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008c0420\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-jrgxnr\" in resourcegroup=\"rg-zstream-upgrade-4-19-5mbcpg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-jrgxnr in resource group rg-zstream-upgrade-4-19-5mbcpg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-jrgxnr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-5mbcpg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-jrgxnr in resource group rg-zstream-upgrade-4-19-5mbcpg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0007fd0b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-jrgxnr in resource group rg-zstream-upgrade-4-19-5mbcpg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"3fc2a1df18a1d6cd12af79418390ec0eeeb5f6b2f096fe3106ec36d30949bf9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"0d24c8a507e608049ef5aef85ec5ea1a9f166d23aa217ff6cdbbc3789165c604","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000696dc0\u003e: \n failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-zdqkwr\" for cluster=\"ea-listmrjvrf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-zdqkwr\\\" for cluster=\\\"ea-listmrjvrf\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f1560\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"166169b8-0520-43f1-8408-4075317b3e22\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234332Z:cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Envoy-Upstream-Service-Time\": [\"238\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/0aa7dca7-ac55-417f-a9ed-45134e5db709\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: A7DD4C4BA3564926A620C3D8ADA15FE4 Ref B: CHI261140511054 Ref C: 2026-04-02T23:43:32Z\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:43:32 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000696d00\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000696dc0\u003e: \n failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-zdqkwr\" for cluster=\"ea-listmrjvrf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-zdqkwr\\\" for cluster=\\\"ea-listmrjvrf\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f1560\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"166169b8-0520-43f1-8408-4075317b3e22\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234332Z:cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Envoy-Upstream-Service-Time\": [\"238\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/0aa7dca7-ac55-417f-a9ed-45134e5db709\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: A7DD4C4BA3564926A620C3D8ADA15FE4 Ref B: CHI261140511054 Ref C: 2026-04-02T23:43:32Z\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:43:32 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000696d00\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"44181b973e6b235efa96b51c10f5880612cfe5ed011abbd0c089fb3fb6030567","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"ff566a0f193f48a9ea966eddaa287843632a1772b7c0ae15a93934857c859a81","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174183 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 00:04:23.463\n \"ts\"=\"2026-04-03 00:04:23.463319\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 00:04:27.591\n STEP: creating customer resources @ 04/03/26 00:04:27.714\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 00:05:20.496\n \"ts\"=\"2026-04-03 00:05:20.496874\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-7s5scz\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 00:18:42.133\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 00:30:11.111\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 00:33:47.481\n STEP: capturing node release images before upgrade @ 04/03/26 00:33:47.596\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 00:33:47.644\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 00:34:39.143\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174183 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 00:04:23.463\n \"ts\"=\"2026-04-03 00:04:23.463319\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 00:04:27.591\n STEP: creating customer resources @ 04/03/26 00:04:27.714\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 00:05:20.496\n \"ts\"=\"2026-04-03 00:05:20.496874\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-7s5scz\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 00:18:42.133\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 00:30:11.111\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 00:33:47.481\n STEP: capturing node release images before upgrade @ 04/03/26 00:33:47.596\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 00:33:47.644\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 00:34:39.143\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"5e568ff00552c6c027f7ec015ef192455bc48648e544be1be607f7326377fa8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"bb803c8293f9cd63e51d5b6e61986b87a54a6d5bac398af30f1214398c7c2f83","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00138d620\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-9tvzvg\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-9tvzvg\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000985ad0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"39fbee8f-3cf5-48e0-b960-f67dbe52bffb\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 61F6A3FD78CC4CD8BFF56C31AA11F0E1 Ref B: CH1AA2020603031 Ref C: 2026-04-02T23:58:09Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:58:09 GMT\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"260\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/90122db5-9edb-46e1-a78d-3e8a94c781aa\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T235809Z:ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00138d560\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00138d620\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-9tvzvg\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-9tvzvg\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000985ad0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"39fbee8f-3cf5-48e0-b960-f67dbe52bffb\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 61F6A3FD78CC4CD8BFF56C31AA11F0E1 Ref B: CH1AA2020603031 Ref C: 2026-04-02T23:58:09Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:58:09 GMT\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"260\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/90122db5-9edb-46e1-a78d-3e8a94c781aa\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T235809Z:ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00138d560\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"730b626f75704093af704b9f0ce747fd162f19348d3d6224b768f3aa091c1e21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"fb486152deca0eba09a175d7862f2d749ccb1ae46463ef20cabad4e7df2b2aaa","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015907b0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tndx4h\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-tndx4h, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-tndx4h\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-tndx4h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e99a40\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-tndx4h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015907b0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tndx4h\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-tndx4h, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-tndx4h\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-tndx4h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e99a40\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-tndx4h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"8084a0d09e6ebe5b72ce9e77b5c98ab1ca2ebe124fc3abb869670a9b0a1d4ac2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"8dd2ce6b458f17f53a3b6d6594cd99e90c9237b7e0dcdd75ea6944a76272b6e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"a54f1c852f9da1d8f55d2f99b680bffc752682b546f9f17e7fc3eb3655aef597","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a06c00\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-p7j8mg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-p7j8mg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-p7j8mg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-p7j8mg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00067e680\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-p7j8mg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a06c00\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-p7j8mg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-p7j8mg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-p7j8mg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-p7j8mg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00067e680\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-p7j8mg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"9944d1d6fc947a53f82449cd765364dca3762e356c2e2ac072a68739e8d545d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"6621bf1de0d69411a9065b77e5668e42eef503bb9ffcc29837584f3701bf7719","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a1cc60\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nm2pxw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-nm2pxw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7e4e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a1cc60\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nm2pxw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-nm2pxw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7e4e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"a448e56f2a6ee55eafc46e0c0088f384a27f4c3e9a3461ceced4f06e5150c21d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"ac746594211fdc7076800aa42ef5b7109df2141917100063844699879227c436","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"a0048cee302965dfeb513ef4fabde7392fc0ee3edd122e97e6c485f5ee38f30f","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a21680\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000845d40\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"154\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:47:08 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"a6270c0d-5904-4199-be64-b4731e31f158\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/25b0928e-3d5f-45fb-ab35-db007159f40d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234708Z:83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 18AE25CEB3C847B2B0880A4661A960C2 Ref B: CHI261140511054 Ref C: 2026-04-02T23:47:08Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a215c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"ma...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a21680\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000845d40\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"154\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:47:08 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"a6270c0d-5904-4199-be64-b4731e31f158\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/25b0928e-3d5f-45fb-ab35-db007159f40d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234708Z:83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 18AE25CEB3C847B2B0880A4661A960C2 Ref B: CHI261140511054 Ref C: 2026-04-02T23:47:08Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a215c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"ma...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bc36242eb1dfc5f7679d5b5a465b272c01b048a94b21fbb3072793ce80c103aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"3bdd4518e35aa05d1f68a1d08e5f52cff965914b56611388669abd719d32cb58","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775173098 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 04/03/26 00:04:23.094\n \"ts\"=\"2026-04-03 00:04:23.094924\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating cluster parameters @ 04/03/26 00:04:26.964\n STEP: creating customer resources @ 04/03/26 00:04:26.964\n STEP: creating the HCP cluster @ 04/03/26 00:05:20.217\n \"ts\"=\"2026-04-03 00:05:20.217661\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating nodepool parameters with autoscaling enabled @ 04/03/26 00:18:41.499\n STEP: creating the autoscaling nodepool @ 04/03/26 00:18:41.499\n STEP: verifying the cluster has default autoscaling parameters @ 04/03/26 00:31:21.659\n \"ts\"=\"2026-04-03 00:31:21.802840\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 00:31:21.802917\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:21.916762\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 00:31:22.918087\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918195\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918256\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:22.919731\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\"\n \"ts\"=\"2026-04-03 00:31:22.919749\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:23.016046\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-04-03 00:31:23.121326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 00:31:23.128889\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\"\n \"ts\"=\"2026-04-03 00:31:23.288779\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"msi-deployment-132\"\n \"ts\"=\"2026-04-03 00:31:23.467072\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:23.467203\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 00:31:23.467319\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078081\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078111\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775173098 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 04/03/26 00:04:23.094\n \"ts\"=\"2026-04-03 00:04:23.094924\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating cluster parameters @ 04/03/26 00:04:26.964\n STEP: creating customer resources @ 04/03/26 00:04:26.964\n STEP: creating the HCP cluster @ 04/03/26 00:05:20.217\n \"ts\"=\"2026-04-03 00:05:20.217661\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating nodepool parameters with autoscaling enabled @ 04/03/26 00:18:41.499\n STEP: creating the autoscaling nodepool @ 04/03/26 00:18:41.499\n STEP: verifying the cluster has default autoscaling parameters @ 04/03/26 00:31:21.659\n \"ts\"=\"2026-04-03 00:31:21.802840\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 00:31:21.802917\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:21.916762\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 00:31:22.918087\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918195\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918256\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:22.919731\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\"\n \"ts\"=\"2026-04-03 00:31:22.919749\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:23.016046\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-04-03 00:31:23.121326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 00:31:23.128889\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\"\n \"ts\"=\"2026-04-03 00:31:23.288779\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"msi-deployment-132\"\n \"ts\"=\"2026-04-03 00:31:23.467072\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:23.467203\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 00:31:23.467319\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078081\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078111\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"bdd7edd823fdba337ab1e898e4edd84d9cd1249cf1b637d07e60175d77f11f3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T00:52:05Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"c37e7e6ef9431683238b22a251accefe1e4fa67f5fb437f88dbf493fe8b4ee58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"2173f70b91c6bd7f5e4e4471684b57b09412d9d569b1fbe8d10eef8d55358155","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174130 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/02/26 23:55:30.12\n \"ts\"=\"2026-04-02 23:55:30.120343\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: creating cluster parameters @ 04/02/26 23:55:34.115\n STEP: creating customer resources @ 04/02/26 23:55:34.116\n STEP: creating the HCP cluster @ 04/02/26 23:56:27.658\n \"ts\"=\"2026-04-02 23:56:27.658606\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: getting credentials @ 04/03/26 00:05:46.538\n STEP: creating the node pool with initial labels and taints @ 04/03/26 00:07:53.808\n STEP: waiting for nodes to be ready @ 04/03/26 00:16:31.838\n STEP: verifying initial labels are present on nodes @ 04/03/26 00:16:31.986\n STEP: verifying initial taints are present on nodes @ 04/03/26 00:16:32.013\n STEP: updating nodepool with new taints and scaling up @ 04/03/26 00:16:32.013\n STEP: verifying nodes are scaled to the expected count @ 04/03/26 00:29:03.022\n STEP: verifying new taint is present on newly created node @ 04/03/26 00:29:03.167\n STEP: updating nodepool with a new label and scaling up @ 04/03/26 00:29:03.194\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174130 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/02/26 23:55:30.12\n \"ts\"=\"2026-04-02 23:55:30.120343\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: creating cluster parameters @ 04/02/26 23:55:34.115\n STEP: creating customer resources @ 04/02/26 23:55:34.116\n STEP: creating the HCP cluster @ 04/02/26 23:56:27.658\n \"ts\"=\"2026-04-02 23:56:27.658606\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: getting credentials @ 04/03/26 00:05:46.538\n STEP: creating the node pool with initial labels and taints @ 04/03/26 00:07:53.808\n STEP: waiting for nodes to be ready @ 04/03/26 00:16:31.838\n STEP: verifying initial labels are present on nodes @ 04/03/26 00:16:31.986\n STEP: verifying initial taints are present on nodes @ 04/03/26 00:16:32.013\n STEP: updating nodepool with new taints and scaling up @ 04/03/26 00:16:32.013\n STEP: verifying nodes are scaled to the expected count @ 04/03/26 00:29:03.022\n STEP: verifying new taint is present on newly created node @ 04/03/26 00:29:03.167\n STEP: updating nodepool with a new label and scaling up @ 04/03/26 00:29:03.194\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d1591c0354ac9489a9551d8f4f4fa75fabba7a83cf648f773e60a072116fc149","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"602da1c819b7275183aaa1febbf8be89b01df162a3578abadddbbccd46f2c239","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dd4e70\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2xbb2m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2xbb2m\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ef9b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dd4e70\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2xbb2m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2xbb2m\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ef9b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"de55929d050aeef8b874ff10b027ae4ff0ae8a01dbcf15708730d5026d1ec909","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","failed_at":"e2e","signature_id":"0a5a5c3cd727714b958f5d99c98b66e348e3db4111454dec8b1404698f2e0794","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012c8540\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-t869kp\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-qnsckw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074f9a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012c8540\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-t869kp\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-qnsckw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074f9a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T21:22:43Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"06e4115c78236a7e57554eeb9f3b12c603bed7bbab474a7235974a309f489afc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"61a0a590b848614b037b56bafbe413c7ebc46d25f3f5fd367783a83492aabae1","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00158e900\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-h9l2tt\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-h9l2tt, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h9l2tt\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-h9l2tt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ff570\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-h9l2tt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00158e900\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-h9l2tt\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-h9l2tt, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-h9l2tt\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-h9l2tt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ff570\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-h9l2tt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"11fed4e40c2117a5955b0aeb84bab883996050e6793bda3bf27f6d45bd577abd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"f41b3019192128e043d558d99fb921bcaaf70089e62332adee984faa6ebca7e5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0017088d0\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-5v4bwn\" in resourcegroup=\"rg-stable-4-20-wttc7h\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5v4bwn in resource group rg-stable-4-20-wttc7h, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-5v4bwn\\\" in resourcegroup=\\\"rg-stable-4-20-wttc7h\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5v4bwn in resource group rg-stable-4-20-wttc7h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ac2e80\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5v4bwn in resource group rg-stable-4-20-wttc7h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0017088d0\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-5v4bwn\" in resourcegroup=\"rg-stable-4-20-wttc7h\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5v4bwn in resource group rg-stable-4-20-wttc7h, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-5v4bwn\\\" in resourcegroup=\\\"rg-stable-4-20-wttc7h\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5v4bwn in resource group rg-stable-4-20-wttc7h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ac2e80\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5v4bwn in resource group rg-stable-4-20-wttc7h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"2f9a278481ab01a44ba3a99737b92a0ccb71713af00dfe0f1e963bd44cf43c89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"8bb7de9b10491a5ec457f50056f87d1f63a31c9cc3e80935e93e3ba4023e2164","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001606750\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-sv2pc6\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-sv2pc6, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-sv2pc6\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-sv2pc6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013a2c10\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-sv2pc6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001606750\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-sv2pc6\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-sv2pc6, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-sv2pc6\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-sv2pc6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013a2c10\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-sv2pc6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"3484ace6d1bfe6f42d2723a3562e843961e20fe180c7b4960dea137b6fd805e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"95d51df2e9bb8e2ab7173a4b7e98f477d5bdd1867f55a5914ac8a5bcf2017b4e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000669920\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-qbtwx2\" in resourcegroup=\"rg-zstream-upgrade-4-19-rdbzd7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-qbtwx2 in resource group rg-zstream-upgrade-4-19-rdbzd7, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-qbtwx2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rdbzd7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-qbtwx2 in resource group rg-zstream-upgrade-4-19-rdbzd7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000af2380\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-qbtwx2 in resource group rg-zstream-upgrade-4-19-rdbzd7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000669920\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-qbtwx2\" in resourcegroup=\"rg-zstream-upgrade-4-19-rdbzd7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-qbtwx2 in resource group rg-zstream-upgrade-4-19-rdbzd7, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-qbtwx2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rdbzd7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-qbtwx2 in resource group rg-zstream-upgrade-4-19-rdbzd7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000af2380\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-qbtwx2 in resource group rg-zstream-upgrade-4-19-rdbzd7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"4c1c8912666d816e05ad0fd74724b4983314843353c85d089dd1c6182344ef99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"c29effccab9c07c6d8bad7280555e5a4daafa065e1f598a5672df600d11435fb","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001406a20\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4s2l8t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4s2l8t, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-4s2l8t\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4s2l8t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000966f70\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4s2l8t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001406a20\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4s2l8t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4s2l8t, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-4s2l8t\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4s2l8t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000966f70\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4s2l8t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"4f8c6ae58b9573d486278a2113a695af17a64df7e936b973b3eb5f051a60a33d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"a2c853e8faaf6b186309d06a6136d2f6b89badfa8f6172ae61fe9adc0c1458c0","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001786690\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thfp4w\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-thfp4w, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-thfp4w\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-thfp4w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001359710\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-thfp4w\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001786690\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thfp4w\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-thfp4w, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-thfp4w\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-thfp4w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001359710\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-thfp4w\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"57688fbb187fed377f9fb53a5b24e8c53179f8fb090ac4d75b1282b9a8f677f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"76a907cf78fc2b98fd4ea961a7d4e7bbd166e544fb45585ee6d37449b8593824","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"49af8d5d6289c5f9653a0cd5b17556a7b822922c916541f0d601e3913728bc43","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000840ca0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-sspgrn-nth9sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-sspgrn/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-sspgrn-nth9sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-sspgrn/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001499d70\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/c0747b09-a0e7-4f8e-8b91-cb4928e98c6c\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"192eab27-e911-4985-9f2c-186922695500\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: F9256E676FE24D3E97CB455F2F63E23F Ref B: CH1AA2020603045 Ref C: 2026-04-02T22:44:02Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 22:44:02 GMT\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"127\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260402T224403Z:192eab27-e911-4985-9f2c-186922695500\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"d3ed7137-b788-458b-a82f-a51611331a6f\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000840be0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n U...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000840ca0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-sspgrn-nth9sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-sspgrn/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-sspgrn-nth9sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-sspgrn/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001499d70\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/c0747b09-a0e7-4f8e-8b91-cb4928e98c6c\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"192eab27-e911-4985-9f2c-186922695500\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: F9256E676FE24D3E97CB455F2F63E23F Ref B: CH1AA2020603045 Ref C: 2026-04-02T22:44:02Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 22:44:02 GMT\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"127\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260402T224403Z:192eab27-e911-4985-9f2c-186922695500\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"d3ed7137-b788-458b-a82f-a51611331a6f\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000840be0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n U...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a4f790db43d06ce9c885342c25236dbb86da848c962919275d89271b4c5e1351","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"bfda7f6f00a6578d4c5d8c5848019b1684cf7315d6a0363d79eb51f8fca100db","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc001010078\u003e: \n found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\n {\n errs: [\n \u003c*errors.errorString | 0xc00185b800\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc001010078\u003e: \n found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\n {\n errs: [\n \u003c*errors.errorString | 0xc00185b800\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"802998fa7e45e8509bd2cc7f1eb9749f5e3559041c2c929960e3ec4481056396","failure_pattern":"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797","search_query":"occurred"}},{"row_id":"a6edec961a8b360a2b70e45b597cbf9f58f09ae04ca43ca04ef6f70c9791a115","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"1ff548cb71a313175df858c7795325be9045905a3111e7e9176defc19836ede5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"16860, 0xc00182ede0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:297 +0x14d fp=0xc000cedfc0 sp=0xc000cedf28 pc=0x527aeed\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:306 +0x28 fp=0xc000cedfe0 sp=0xc000cedfc0 pc=0x527ad68\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000cedfe8 sp=0xc000cedfe0 pc=0x2dbecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 793\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:290 +0xc5\n\ngoroutine 772 gp=0xc000ecddc0 m=nil [IO wait]:\nruntime.gopark(0xc0006328c0?, 0xb0ed2000?, 0x50?, 0x37?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000cf3720 sp=0xc000cf3700 pc=0x2db650e\nruntime.netpollblock(0x2e20d18?, 0x2d42066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000cf3758 sp=0xc000cf3720 pc=0x2d75fb7\ninternal/poll.runtime_pollWait(0x7f9fb2f96600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000cf3778 sp=0xc000cf3758 pc=0x2db4da5\ninternal/poll.(*pollDesc).wait(0xc0008b2d00?, 0xc0011ac000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000cf37a0 sp=0xc000cf3778 pc=0x2e43a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0008b2d00, {0xc0011ac000, 0x6000, 0x6000})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000cf3838 sp=0xc000cf37a0 pc=0x2e44e79\nnet.(*netFD).Read(0xc0008b2d00, {0xc0011ac000?, 0xc0011ac2f4?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000cf3880 sp=0xc000cf3838 pc=0x300a4e5\nnet.(*conn).Read(0xc0001aa328, {0xc0011ac000?, 0x7f9fb0ed1800?, 0x7f9ff9b7d5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000cf38c8 sp=0xc000cf3880 pc=0x301c585\ncrypto/tls.(*atLeastReader).Read(0xc0018c0330, {0xc0011ac000?, 0x5d07?, 0x2db3245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000cf3910 sp=0xc000cf38c8 pc=0x308801b\nbytes.(*Buffer).ReadFrom(0xc0007c77a8, {0x8c11b40, 0xc0018c0330})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000cf3968 sp=0xc000cf3910 pc=0x2e81c78\ncrypto/tls.(*Conn).readFromUntil(0xc0007c7508, {0x8c12920, 0xc0001aa328}, 0xc000cf39d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000cf39a0 sp=0xc000cf3968 pc=0x30881fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0007c7508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000cf3c18 sp=0xc000cf39a0 pc=0x308527b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0007c7508, {0xc001131000, 0x1000, 0x34ca980?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000cf3c80 sp=0xc000cf3c18 pc=0x308bb65\nbufio.(*Reader).Read(0xc0010f70e0, {0xc000770824, 0x9, 0x34d8a2e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000cf3cb8 sp=0xc000cf3c80 pc=0x2f1e1d7\nio.ReadAtLeast({0x8c11140, 0xc0010f70e0}, {0xc000770824, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000cf3d00 sp=0xc000cf3cb8 pc=0x2e10e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000770824, 0x9, 0xc000000069?}, {0x8c11140?, 0xc0010f70e0?})\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:250 +0x65 fp=0xc000cf3d50 sp=0xc000cf3d00 pc=0x34c1745\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0007707e0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:513 +0x6b fp=0xc000cf3dd8 sp=0xc000cf3d50 pc=0x34c230b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0007707e0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:572 +0x18 fp=0xc000cf3e18 sp=0xc000cf3dd8 pc=0x34c2a78\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000cf3fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2254 +0xca fp=0xc000cf3f60 sp=0xc000cf3e18 pc=0x34d596a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0010f8380)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2123 +0x52 fp=0xc000cf3fc8 sp=0xc000cf3f60 pc=0x34d4cf2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0x25 fp=0xc000cf3fe0 sp=0xc000cf3fc8 pc=0x34ce485\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000cf3fe8 sp=0xc000cf3fe0 pc=0x2dbecc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 771\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0xda5\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1ba5b0\nrsi 0x80\nrbp 0x7ffd3f8d97b8\nrsp 0x7ffd3f8d9770\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x6a\nr13 0x1\nr14 0xc1b84c0\nr15 0x5\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n16860, 0xc00182ede0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:297 +0x14d fp=0xc000cedfc0 sp=0xc000cedf28 pc=0x527aeed\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:306 +0x28 fp=0xc000cedfe0 sp=0xc000cedfc0 pc=0x527ad68\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000cedfe8 sp=0xc000cedfe0 pc=0x2dbecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 793\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:290 +0xc5\n\ngoroutine 772 gp=0xc000ecddc0 m=nil [IO wait]:\nruntime.gopark(0xc0006328c0?, 0xb0ed2000?, 0x50?, 0x37?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000cf3720 sp=0xc000cf3700 pc=0x2db650e\nruntime.netpollblock(0x2e20d18?, 0x2d42066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000cf3758 sp=0xc000cf3720 pc=0x2d75fb7\ninternal/poll.runtime_pollWait(0x7f9fb2f96600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000cf3778 sp=0xc000cf3758 pc=0x2db4da5\ninternal/poll.(*pollDesc).wait(0xc0008b2d00?, 0xc0011ac000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000cf37a0 sp=0xc000cf3778 pc=0x2e43a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0008b2d00, {0xc0011ac000, 0x6000, 0x6000})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000cf3838 sp=0xc000cf37a0 pc=0x2e44e79\nnet.(*netFD).Read(0xc0008b2d00, {0xc0011ac000?, 0xc0011ac2f4?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000cf3880 sp=0xc000cf3838 pc=0x300a4e5\nnet.(*conn).Read(0xc0001aa328, {0xc0011ac000?, 0x7f9fb0ed1800?, 0x7f9ff9b7d5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000cf38c8 sp=0xc000cf3880 pc=0x301c585\ncrypto/tls.(*atLeastReader).Read(0xc0018c0330, {0xc0011ac000?, 0x5d07?, 0x2db3245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000cf3910 sp=0xc000cf38c8 pc=0x308801b\nbytes.(*Buffer).ReadFrom(0xc0007c77a8, {0x8c11b40, 0xc0018c0330})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000cf3968 sp=0xc000cf3910 pc=0x2e81c78\ncrypto/tls.(*Conn).readFromUntil(0xc0007c7508, {0x8c12920, 0xc0001aa328}, 0xc000cf39d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000cf39a0 sp=0xc000cf3968 pc=0x30881fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0007c7508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000cf3c18 sp=0xc000cf39a0 pc=0x308527b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0007c7508, {0xc001131000, 0x1000, 0x34ca980?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000cf3c80 sp=0xc000cf3c18 pc=0x308bb65\nbufio.(*Reader).Read(0xc0010f70e0, {0xc000770824, 0x9, 0x34d8a2e?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000cf3cb8 sp=0xc000cf3c80 pc=0x2f1e1d7\nio.ReadAtLeast({0x8c11140, 0xc0010f70e0}, {0xc000770824, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000cf3d00 sp=0xc000cf3cb8 pc=0x2e10e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000770824, 0x9, 0xc000000069?}, {0x8c11140?, 0xc0010f70e0?})\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:250 +0x65 fp=0xc000cf3d50 sp=0xc000cf3d00 pc=0x34c1745\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0007707e0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:513 +0x6b fp=0xc000cf3dd8 sp=0xc000cf3d50 pc=0x34c230b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0007707e0)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/frame.go:572 +0x18 fp=0xc000cf3e18 sp=0xc000cf3dd8 pc=0x34c2a78\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000cf3fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2254 +0xca fp=0xc000cf3f60 sp=0xc000cf3e18 pc=0x34d596a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0010f8380)\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:2123 +0x52 fp=0xc000cf3fc8 sp=0xc000cf3f60 pc=0x34d4cf2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0x25 fp=0xc000cf3fe0 sp=0xc000cf3fc8 pc=0x34ce485\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000cf3fe8 sp=0xc000cf3fe0 pc=0x2dbecc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 771\n\t/go/pkg/mod/golang.org/x/net@v0.52.0/http2/transport.go:876 +0xda5\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1ba5b0\nrsi 0x80\nrbp 0x7ffd3f8d97b8\nrsp 0x7ffd3f8d9770\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x6a\nr13 0x1\nr14 0xc1b84c0\nr15 0x5\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d8fb2fa8a3b520a82f98f2794f344abd1e5929de44bde6f812c3b6ca61bc55bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"f2e9ab95d59de09cd7c68b9d45603f48dcc7ae70fd06bf026e6a73c00e33f05c","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e2de0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-tmq76j\" in resourcegroup=\"admin-credential-lifecycle-test-fvb2sn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-tmq76j in resource group admin-credential-lifecycle-test-fvb2sn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tmq76j\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-fvb2sn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-tmq76j in resource group admin-credential-lifecycle-test-fvb2sn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001575f50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-tmq76j in resource group admin-credential-lifecycle-test-fvb2sn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e2de0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-tmq76j\" in resourcegroup=\"admin-credential-lifecycle-test-fvb2sn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-tmq76j in resource group admin-credential-lifecycle-test-fvb2sn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-tmq76j\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-fvb2sn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-tmq76j in resource group admin-credential-lifecycle-test-fvb2sn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001575f50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-tmq76j in resource group admin-credential-lifecycle-test-fvb2sn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"e5190e953659e786ded79927a771aa65a85ce8098162ff737b5e1ade4d2d5417","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","failed_at":"e2e","signature_id":"70c3aed9dcf9d4690d316f75b5f6293fb9b255dea6bc67906dc9905abcacb821","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e80390\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-dh4k5t\" in resourcegroup=\"rg-zstream-upgrade-4-21-5xwxz7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-dh4k5t in resource group rg-zstream-upgrade-4-21-5xwxz7, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-dh4k5t\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-5xwxz7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-dh4k5t in resource group rg-zstream-upgrade-4-21-5xwxz7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006a10b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-dh4k5t in resource group rg-zstream-upgrade-4-21-5xwxz7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e80390\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-dh4k5t\" in resourcegroup=\"rg-zstream-upgrade-4-21-5xwxz7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-dh4k5t in resource group rg-zstream-upgrade-4-21-5xwxz7, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-dh4k5t\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-5xwxz7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-dh4k5t in resource group rg-zstream-upgrade-4-21-5xwxz7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006a10b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-dh4k5t in resource group rg-zstream-upgrade-4-21-5xwxz7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","802998fa7e45e8509bd2cc7f1eb9749f5e3559041c2c929960e3ec4481056396","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T20:26:55Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"031761ebb5a8c2fc952d2295d7203749fec3b376bca230e5cc532ad17500fb74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"3994952cbbf87a51591ede9e6c056fa912c759e66404e2786708b117d1988eb5","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dbeed0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4v6pcw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4v6pcw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-4v6pcw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4v6pcw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a57300\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4v6pcw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dbeed0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4v6pcw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4v6pcw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-4v6pcw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4v6pcw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a57300\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4v6pcw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"794516f12e1b42de5e1a6d5ce59332df755aa3016e401376175733d6725a0767","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"f1e0d24bd3fa73c5580d0a8e6922db4fef593d3a6d46793c243e5e71c5f53b67","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010bac80\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hzrfhf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hzrfhf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001203e30\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"e1ea6c99-e1af-4578-aef0-1306b523c906\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"2d2345a2-c74d-4312-a848-a40bf84bd584\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"841\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/7287e47d-3f8a-4d8b-8e48-1036f8537975\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 99D6B712DDED4198801BF5D06DC67B75 Ref B: CHI261140501040 Ref C: 2026-04-02T21:15:54Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 21:15:54 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T211555Z:e1ea6c99-e1af-4578-aef0-1306b523c906\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0010babc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010bac80\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hzrfhf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hzrfhf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001203e30\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"107\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"e1ea6c99-e1af-4578-aef0-1306b523c906\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"2d2345a2-c74d-4312-a848-a40bf84bd584\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"841\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/7287e47d-3f8a-4d8b-8e48-1036f8537975\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 99D6B712DDED4198801BF5D06DC67B75 Ref B: CHI261140501040 Ref C: 2026-04-02T21:15:54Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 21:15:54 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T211555Z:e1ea6c99-e1af-4578-aef0-1306b523c906\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0010babc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8978c72161c466e237f2990ae3098709d8e05ca7e7fc149b5780efef4b51820f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"ae436fa9181fe834f84e3b1d788442c6f87e6b4b936c84cd5b8a798d29187869","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"4c7844e62748bc91078d395584d560c1e7d7170e1afc3d5c2015d760a9626096","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a56060\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-vvkmt7\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-vvkmt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-vvkmt7\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-vvkmt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000959e60\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"602\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"3bba6e4b-62fc-4679-ac86-6cca778ca60d\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: E927148DA18943C493DC7980C427DBBF Ref B: CHI261140511042 Ref C: 2026-04-02T21:28:00Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"c4a7fd9a-75bf-42b2-9162-e851d965413a\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/6c5a5ff0-966d-49cb-9120-62d22bd412e9\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T212801Z:3bba6e4b-62fc-4679-ac86-6cca778ca60d\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 21:28:00 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b1df20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a56060\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-vvkmt7\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-vvkmt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-vvkmt7\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-vvkmt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000959e60\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"602\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"3bba6e4b-62fc-4679-ac86-6cca778ca60d\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: E927148DA18943C493DC7980C427DBBF Ref B: CHI261140511042 Ref C: 2026-04-02T21:28:00Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"c4a7fd9a-75bf-42b2-9162-e851d965413a\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/6c5a5ff0-966d-49cb-9120-62d22bd412e9\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T212801Z:3bba6e4b-62fc-4679-ac86-6cca778ca60d\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 21:28:00 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000b1df20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b1811dd032f89d10571dd6da6c9df116e13ecaea8493c9aee4ac174c15aee252","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"44c962b6b7ee4eda614e728b25fe72f7f819bd96a0eafa86267710f755d3954d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015b4ff0\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-cjp2ct\" in resourcegroup=\"rg-zstream-upgrade-4-20-92d5lx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-cjp2ct in resource group rg-zstream-upgrade-4-20-92d5lx, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-cjp2ct\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-92d5lx\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-cjp2ct in resource group rg-zstream-upgrade-4-20-92d5lx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001687e10\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-cjp2ct in resource group rg-zstream-upgrade-4-20-92d5lx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015b4ff0\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-cjp2ct\" in resourcegroup=\"rg-zstream-upgrade-4-20-92d5lx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-cjp2ct in resource group rg-zstream-upgrade-4-20-92d5lx, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-cjp2ct\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-92d5lx\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-cjp2ct in resource group rg-zstream-upgrade-4-20-92d5lx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001687e10\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-cjp2ct in resource group rg-zstream-upgrade-4-20-92d5lx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"dbf5e0a295c3f51426a5087e0b67d9da94d0f65a903fcb01b3959446660505e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"54c9710ba8898fedef07e0e373e1e9bec177d9d7a3a262f4872e5a0e23e67e76","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f5aa80\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-xn9npb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-xn9npb, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-xn9npb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-xn9npb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008e33b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-xn9npb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f5aa80\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-xn9npb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-xn9npb, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-xn9npb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-xn9npb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008e33b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-xn9npb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"df3ab772ea75bf8a68669266a86d09cfa26a66350ae5a3748fc2927078bcc17f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"d478f3c070bdbbdf6cce6725f9b35024fea57ed2435ef8346a8feef92e7f5381","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075a160\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-stable-4-20-zchh4x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-stable-4-20-gcjhh8/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-stable-4-20-zchh4x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-stable-4-20-gcjhh8/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001206570\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T214410Z:4a38c176-8867-44b2-a3c7-48492a8d4e0f\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 79D0092FC1DF43CBAC048BA3F59FDC4E Ref B: CHI261140505040 Ref C: 2026-04-02T21:44:10Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"cb3209b7-3062-4e31-95f1-095a20aeff07\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/c6802a55-7b52-4d68-ac18-173dfd506998\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 21:44:10 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"219\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"4a38c176-8867-44b2-a3c7-48492a8d4e0f\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00075a060\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075a160\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-stable-4-20-zchh4x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-stable-4-20-gcjhh8/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-stable-4-20-zchh4x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-stable-4-20-gcjhh8/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001206570\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T214410Z:4a38c176-8867-44b2-a3c7-48492a8d4e0f\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 79D0092FC1DF43CBAC048BA3F59FDC4E Ref B: CHI261140505040 Ref C: 2026-04-02T21:44:10Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"cb3209b7-3062-4e31-95f1-095a20aeff07\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/c6802a55-7b52-4d68-ac18-173dfd506998\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 21:44:10 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"219\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"4a38c176-8867-44b2-a3c7-48492a8d4e0f\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00075a060\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4652577934e838d85dd1beea87da2e17bf1da5072a331019e59c273da23230b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"a693b54b61382ae49d313d16b61863448c5c697a2d7e8a751f27a72c483613c4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001568360\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-45hj5h\" in resourcegroup=\"rg-zstream-upgrade-4-19-g7skvb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-45hj5h in resource group rg-zstream-upgrade-4-19-g7skvb, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-45hj5h\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-g7skvb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-45hj5h in resource group rg-zstream-upgrade-4-19-g7skvb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010628b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-45hj5h in resource group rg-zstream-upgrade-4-19-g7skvb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001568360\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-45hj5h\" in resourcegroup=\"rg-zstream-upgrade-4-19-g7skvb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-45hj5h in resource group rg-zstream-upgrade-4-19-g7skvb, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-45hj5h\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-g7skvb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-45hj5h in resource group rg-zstream-upgrade-4-19-g7skvb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010628b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-45hj5h in resource group rg-zstream-upgrade-4-19-g7skvb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"f57395f6205fc1173f3771e9d9fbdfdfab573ab64fea4366de7515f94ff97e8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","failed_at":"e2e","signature_id":"a5acc838efa40f2e3015b2f5656215fc959732bf821c4f98925ba8e8460b95e5","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000c1a9f0\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-kldf9f\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-kldf9f, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-kldf9f\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-kldf9f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015c97d0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-kldf9f\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000c1a9f0\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-kldf9f\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-kldf9f, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-kldf9f\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-kldf9f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015c97d0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-kldf9f\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T19:11:48Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"88aa34ef5c5bf9700158247918d75a2c3ac088f0a82c4868d89dfa68fcbd38b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","failed_at":"e2e","signature_id":"69b97748a98721d03376b047e26a8ab738922dee3e3915026ee9110e4927c1ad","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc00173bb78 sp=0xc00173bb58 pc=0x3af095e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters({0x7f6251aa1260, 0xc000c5c210}, 0xc000de52a8, {0xc000d81c20, 0x1e}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:352 +0x8ef fp=0xc00173be20 sp=0xc00173bb78 pc=0x525e40f\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc0009a4780, {0x7f6251aa1260, 0xc000c5c210}, {0xc000d81c20, 0x1e}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:436 +0x294 fp=0xc00173bf28 sp=0xc00173be20 pc=0x527c3b4\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f6251aa1260, 0xc000c5c210})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:297 +0x14d fp=0xc00173bfc0 sp=0xc00173bf28 pc=0x527aeed\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:306 +0x28 fp=0xc00173bfe0 sp=0xc00173bfc0 pc=0x527ad68\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00173bfe8 sp=0xc00173bfe0 pc=0x2dbecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 589\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:290 +0xc5\n\ngoroutine 589 gp=0xc001544e00 m=nil [sync.WaitGroup.Wait, 5 minutes]:\nruntime.gopark(0xc000da95f0?, 0x2d4d874?, 0x60?, 0x6f?, 0x200c0016309a0?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000c65590 sp=0xc000c65570 pc=0x2db650e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc00118f8c8, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000c655f8 sp=0xc000c65590 pc=0x2d9259d\nsync.runtime_SemacquireWaitGroup(0xc0007953a0?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000c65630 sp=0xc000c655f8 pc=0x2db7c2e\nsync.(*WaitGroup).Wait(0xc00118f8c0)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000c65658 sp=0xc000c65630 pc=0x2ddb1a5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc0009a4780, {0x7f6251aa1260, 0xc000c5c210}, {{0xc00097ad80, 0x1, 0x1}, 0x34630b8a000, {0x81c7dce, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:308 +0x273 fp=0xc000c65768 sp=0xc000c65658 pc=0x527a9b3\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc0009a4780, {0x7f6251aa1260, 0xc000c5c210})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:227 +0x225 fp=0xc000c65888 sp=0xc000c65768 pc=0x527a245\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6251aa1260?, 0xc000c5c210?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000c658b0 sp=0xc000c65888 pc=0x528ba73\nruntime.call16(0xc00147f710, 0xc000b04760, 0x0, 0x0, 0x0, 0x10, 0xc000c65b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000c658d0 sp=0xc000c658b0 pc=0x2dbd223\nruntime.reflectcall(0x7bc5d80?, 0xc000c5c210?, 0x2?, 0x81fbd86?, 0x0?, 0x12?, 0x77c4e60?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000c65910 sp=0xc000c658d0 pc=0x2dc5ab6\nreflect.Value.call({0x70d3280?, 0xc000b04760?, 0xc1b6f40?}, {0x81bbe44, 0x4}, {0xc00010df60, 0x1, 0xc000b98388?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000c65e70 sp=0xc000c65910 pc=0x2df8a86\nreflect.Value.Call({0x70d3280?, 0xc000b04760?, 0x0?}, {0xc00010df60?, 0xc00010de50?, 0xc00010df68?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000c65ee8 sp=0xc000c65e70 pc=0x2df7b79\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x8cc8ce0?, 0xc000c5c210?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000c65f88 sp=0xc000c65ee8 pc=0x321aff0\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000c65fe0 sp=0xc000c65f88 pc=0x322dabb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000c65fe8 sp=0xc000c65fe0 pc=0x2dbecc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 86\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1ba5b0\nrsi 0x80\nrbp 0x7ffe92ed0ff0\nrsp 0x7ffe92ed0fa8\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x0\nr13 0x1\nr14 0xc1b84c0\nr15 0x7\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc00173bb78 sp=0xc00173bb58 pc=0x3af095e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters({0x7f6251aa1260, 0xc000c5c210}, 0xc000de52a8, {0xc000d81c20, 0x1e}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:352 +0x8ef fp=0xc00173be20 sp=0xc00173bb78 pc=0x525e40f\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc0009a4780, {0x7f6251aa1260, 0xc000c5c210}, {0xc000d81c20, 0x1e}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:436 +0x294 fp=0xc00173bf28 sp=0xc00173be20 pc=0x527c3b4\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f6251aa1260, 0xc000c5c210})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:297 +0x14d fp=0xc00173bfc0 sp=0xc00173bf28 pc=0x527aeed\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:306 +0x28 fp=0xc00173bfe0 sp=0xc00173bfc0 pc=0x527ad68\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00173bfe8 sp=0xc00173bfe0 pc=0x2dbecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 589\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:290 +0xc5\n\ngoroutine 589 gp=0xc001544e00 m=nil [sync.WaitGroup.Wait, 5 minutes]:\nruntime.gopark(0xc000da95f0?, 0x2d4d874?, 0x60?, 0x6f?, 0x200c0016309a0?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000c65590 sp=0xc000c65570 pc=0x2db650e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc00118f8c8, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc000c655f8 sp=0xc000c65590 pc=0x2d9259d\nsync.runtime_SemacquireWaitGroup(0xc0007953a0?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc000c65630 sp=0xc000c655f8 pc=0x2db7c2e\nsync.(*WaitGroup).Wait(0xc00118f8c0)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc000c65658 sp=0xc000c65630 pc=0x2ddb1a5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc0009a4780, {0x7f6251aa1260, 0xc000c5c210}, {{0xc00097ad80, 0x1, 0x1}, 0x34630b8a000, {0x81c7dce, 0x8}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:308 +0x273 fp=0xc000c65768 sp=0xc000c65658 pc=0x527a9b3\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc0009a4780, {0x7f6251aa1260, 0xc000c5c210})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:227 +0x225 fp=0xc000c65888 sp=0xc000c65768 pc=0x527a245\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6251aa1260?, 0xc000c5c210?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc000c658b0 sp=0xc000c65888 pc=0x528ba73\nruntime.call16(0xc00147f710, 0xc000b04760, 0x0, 0x0, 0x0, 0x10, 0xc000c65b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc000c658d0 sp=0xc000c658b0 pc=0x2dbd223\nruntime.reflectcall(0x7bc5d80?, 0xc000c5c210?, 0x2?, 0x81fbd86?, 0x0?, 0x12?, 0x77c4e60?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000c65910 sp=0xc000c658d0 pc=0x2dc5ab6\nreflect.Value.call({0x70d3280?, 0xc000b04760?, 0xc1b6f40?}, {0x81bbe44, 0x4}, {0xc00010df60, 0x1, 0xc000b98388?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000c65e70 sp=0xc000c65910 pc=0x2df8a86\nreflect.Value.Call({0x70d3280?, 0xc000b04760?, 0x0?}, {0xc00010df60?, 0xc00010de50?, 0xc00010df68?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000c65ee8 sp=0xc000c65e70 pc=0x2df7b79\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x8cc8ce0?, 0xc000c5c210?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc000c65f88 sp=0xc000c65ee8 pc=0x321aff0\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000c65fe0 sp=0xc000c65f88 pc=0x322dabb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000c65fe8 sp=0xc000c65fe0 pc=0x2dbecc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 86\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1ba5b0\nrsi 0x80\nrbp 0x7ffe92ed0ff0\nrsp 0x7ffe92ed0fa8\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x0\nr13 0x1\nr14 0xc1b84c0\nr15 0x7\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"9369361a70ad61748246cbea36b008bff2123d03627564331a5ee90f869d8439","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","failed_at":"e2e","signature_id":"3dbc76b005b4e15f86bd5dc3fdc4f60aceb0cbe928754717e37b1fa61707d06e","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001359200\u003e: \n failed waiting for hcpCluster=\"np-version-upgrade-cluster-6zntpn\" in resourcegroup=\"rg-np-version-upgrade-6zntpn-9zhbdp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-6zntpn in resource group rg-np-version-upgrade-6zntpn-9zhbdp, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"np-version-upgrade-cluster-6zntpn\\\" in resourcegroup=\\\"rg-np-version-upgrade-6zntpn-9zhbdp\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-6zntpn in resource group rg-np-version-upgrade-6zntpn-9zhbdp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000927ca0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-6zntpn in resource group rg-np-version-upgrade-6zntpn-9zhbdp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001359200\u003e: \n failed waiting for hcpCluster=\"np-version-upgrade-cluster-6zntpn\" in resourcegroup=\"rg-np-version-upgrade-6zntpn-9zhbdp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-6zntpn in resource group rg-np-version-upgrade-6zntpn-9zhbdp, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"np-version-upgrade-cluster-6zntpn\\\" in resourcegroup=\\\"rg-np-version-upgrade-6zntpn-9zhbdp\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-6zntpn in resource group rg-np-version-upgrade-6zntpn-9zhbdp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000927ca0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-6zntpn in resource group rg-np-version-upgrade-6zntpn-9zhbdp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"96cdec38581a2240f0a3964d6bbb3d73cdd2e00caa7e14ac6855b518f4cfa1f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","failed_at":"e2e","signature_id":"17372c8fccbca8cace580c5893ad858a6d2524fe5aee1dcf6598b8835c3a2b20","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008d4e70\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8qgh84\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-8qgh84, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-8qgh84\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-8qgh84, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d5cc0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-8qgh84\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008d4e70\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8qgh84\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-8qgh84, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-8qgh84\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-8qgh84, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d5cc0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-8qgh84\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"a5aeb5d5c236051d179f6bfb5ed5b3566aebd7fb88e02e2a10b32ddb9af2463b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","failed_at":"e2e","signature_id":"96ca622f1e934bff69e33dc1c686d3fd304f04accb666698f76f6d12378d24f4","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014890b0\u003e: \n failed waiting for hcpCluster=\"np-version-upgrade-cluster-99z6nv\" in resourcegroup=\"rg-np-version-upgrade-99z6nv-2gt94h\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-99z6nv in resource group rg-np-version-upgrade-99z6nv-2gt94h, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"np-version-upgrade-cluster-99z6nv\\\" in resourcegroup=\\\"rg-np-version-upgrade-99z6nv-2gt94h\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-99z6nv in resource group rg-np-version-upgrade-99z6nv-2gt94h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014706c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-99z6nv in resource group rg-np-version-upgrade-99z6nv-2gt94h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014890b0\u003e: \n failed waiting for hcpCluster=\"np-version-upgrade-cluster-99z6nv\" in resourcegroup=\"rg-np-version-upgrade-99z6nv-2gt94h\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-99z6nv in resource group rg-np-version-upgrade-99z6nv-2gt94h, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"np-version-upgrade-cluster-99z6nv\\\" in resourcegroup=\\\"rg-np-version-upgrade-99z6nv-2gt94h\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-99z6nv in resource group rg-np-version-upgrade-99z6nv-2gt94h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014706c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-99z6nv in resource group rg-np-version-upgrade-99z6nv-2gt94h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded"}},{"row_id":"a7101f690734366a37fe11f7880cabd9abc1fb85efa6aef160de6165ffd7ad81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","failed_at":"e2e","signature_id":"a24f878cc7a5f55b9e221a3806e22a12eb532f19da6bbd44a0b8d98d0ae9e30b","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"rupted by User\n First interrupt received; Ginkgo will run any cleanup and reporting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to create an HCP cluster and custom node pool osDisk size (Spec Runtime: 1h29m59.811s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n In [It] (Node Runtime: 1h29m59.811s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n At [By Step] getting credentials (Step Runtime: 3m7.813s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96\n\n Spec Goroutine\n goroutine 132 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8cb0998, 0xc000530fc0}, 0x81dda58?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cfd5c0, {0x8cb0998, 0xc000530fc0}, 0xc000d10af0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetAdminRESTConfigForHCPCluster(0xc0007f3cc0, {0x7f25e426ce60, 0xc0009475f0}, 0xc000df1f30, {0xc0016121f8, 0x13}, {0x81fbb34, 0x12}, 0x8bb2c97000)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:107\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func9.2({0x7f25e426ce60, 0xc0009475f0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:97\n | \n | By(\"getting credentials\")\n \u003e adminRESTConfig, err := tc.GetAdminRESTConfigForHCPCluster(\n | \tctx,\n | \ttc.Get20240610ClientFactoryOrDie(ctx).NewHcpOpenShiftClustersClient(),\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc8ce0?, 0xc0009475f0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 129\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104 @ 04/02/26 20:55:25.21\n \"ts\"=\"2026-04-02 20:55:25.211124\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 20:55:25.211230\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pjn74q\" \"resourceGroup\"=\"clusternp128-7zlf92\"\n \"ts\"=\"2026-04-02 20:55:25.489641\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-pjn74q\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 20:55:28.102668\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-04-02 20:55:28.102788\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-04-02 20:55:28.102850\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 20:55:28.104321\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\"\n \"ts\"=\"2026-04-02 20:55:28.104333\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-7zlf92\"\n \"ts\"=\"2026-04-02 20:55:28.256263\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 20:55:28.426031\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-7zlf92\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n \"ts\"=\"2026-04-02 20:55:28.560024\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"msi-deployment-78\"\n \"ts\"=\"2026-04-02 20:55:28.696547\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-7zlf92\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pjn74q\"\n \"ts\"=\"2026-04-02 20:55:28.998126\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 20:55:28.998270\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 20:55:28.998395\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-7zlf92\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nrupted by User\n First interrupt received; Ginkgo will run any cleanup and reporting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to create an HCP cluster and custom node pool osDisk size (Spec Runtime: 1h29m59.811s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n In [It] (Node Runtime: 1h29m59.811s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n At [By Step] getting credentials (Step Runtime: 3m7.813s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96\n\n Spec Goroutine\n goroutine 132 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8cb0998, 0xc000530fc0}, 0x81dda58?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cfd5c0, {0x8cb0998, 0xc000530fc0}, 0xc000d10af0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetAdminRESTConfigForHCPCluster(0xc0007f3cc0, {0x7f25e426ce60, 0xc0009475f0}, 0xc000df1f30, {0xc0016121f8, 0x13}, {0x81fbb34, 0x12}, 0x8bb2c97000)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:107\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func9.2({0x7f25e426ce60, 0xc0009475f0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:97\n | \n | By(\"getting credentials\")\n \u003e adminRESTConfig, err := tc.GetAdminRESTConfigForHCPCluster(\n | \tctx,\n | \ttc.Get20240610ClientFactoryOrDie(ctx).NewHcpOpenShiftClustersClient(),\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc8ce0?, 0xc0009475f0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 129\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104 @ 04/02/26 20:55:25.21\n \"ts\"=\"2026-04-02 20:55:25.211124\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 20:55:25.211230\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pjn74q\" \"resourceGroup\"=\"clusternp128-7zlf92\"\n \"ts\"=\"2026-04-02 20:55:25.489641\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-pjn74q\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 20:55:28.102668\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-04-02 20:55:28.102788\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n \"ts\"=\"2026-04-02 20:55:28.102850\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 20:55:28.104321\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\"\n \"ts\"=\"2026-04-02 20:55:28.104333\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-7zlf92\"\n \"ts\"=\"2026-04-02 20:55:28.256263\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 20:55:28.426031\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-7zlf92\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n \"ts\"=\"2026-04-02 20:55:28.560024\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"msi-deployment-78\"\n \"ts\"=\"2026-04-02 20:55:28.696547\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-7zlf92\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pjn74q\"\n \"ts\"=\"2026-04-02 20:55:28.998126\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 20:55:28.998270\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 20:55:28.998395\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-7zlf92\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T18:38:31Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7884b3b4d2766c55d37f52b4a0b462fe19d258f6aee90cbc38d57987bb38c395","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","occurred_at":"2026-04-02T18:38:31Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"d0719b677acd7689e6f97c440c3454f7c912a7640f1dce13d07d702a783fbaa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","occurred_at":"2026-04-02T18:38:31Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T18:07:57Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"ae06d4eacab28395a2df9af229804ec2a6bb6238bfa251879cc9bfa1ada3e4a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"ba4677d2e746388a157a39263571c37577eda2dc8539a90449ebadcd921aa399","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","failed_at":"e2e","signature_id":"0d6d738f471638f58110a043a102f05c2da76aaba189c4a5e7aa46163da27f0d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"ting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API (Spec Runtime: 1h29m59.877s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42\n In [It] (Node Runtime: 1h29m59.877s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42\n At [By Step] getting admin credentials for the cluster (Step Runtime: 5m38.233s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:157\n\n Spec Goroutine\n goroutine 87 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8cb0998, 0xc000a76850}, 0x81dda58?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cfd5c0, {0x8cb0998, 0xc000a76850}, 0xc000a449f0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetAdminRESTConfigForHCPCluster(0xc000b14640, {0x7fa0219e8860, 0xc000c710e0}, 0xc0000d9ec8, {0xc001810108, 0x17}, {0x81fbb46, 0x12}, 0x8bb2c97000)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:107\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func10.2({0x7fa0219e8860, 0xc000c710e0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:158\n | \n | By(\"getting admin credentials for the cluster\")\n \u003e adminRESTConfig, err := tc.GetAdminRESTConfigForHCPCluster(\n | \tctx,\n | \ttc.Get20240610ClientFactoryOrDie(ctx).NewHcpOpenShiftClustersClient(),\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc8ce0?, 0xc000c710e0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 40\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:165 @ 04/02/26 19:42:28.688\n \"ts\"=\"2026-04-02 19:42:28.689138\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 19:42:28.689210\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-private-kv-cluster-brmrkf\" \"resourceGroup\"=\"private-keyvault-w6qm7t\"\n \"ts\"=\"2026-04-02 19:42:28.802742\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-private-kv-cluster-brmrkf\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 19:42:29.228630\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-65e261cf588b9f2637c69902ef65386ae7e651e35485a39612d668d6a3309cde.yaml.gz\"\n \"ts\"=\"2026-04-02 19:42:29.228771\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-65e261cf588b9f2637c69902ef65386ae7e651e35485a39612d668d6a3309cde.yaml.gz\"\n \"ts\"=\"2026-04-02 19:42:29.228877\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 19:42:29.231553\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-18\"\n \"ts\"=\"2026-04-02 19:42:29.231567\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"private-keyvault-w6qm7t\"\n \"ts\"=\"2026-04-02 19:42:29.323497\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"private-keyvault-w6qm7t\" \"deployment\"=\"pooledNonMsiScopedAssignments-had2isw6vkgre\"\n \"ts\"=\"2026-04-02 19:42:29.375537\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-18\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 19:42:29.376346\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"private-keyvault-w6qm7t\" \"deployment\"=\"customer-infra-private-kv-cluster-brmrkf\"\n \"ts\"=\"2026-04-02 19:42:29.521817\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-18\" \"deployment\"=\"msi-deployment-18\"\n \"ts\"=\"2026-04-02 19:42:29.680009\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 19:42:29.680123\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 19:42:29.680252\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"private-keyvault-w6qm7t\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API (Spec Runtime: 1h29m59.877s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42\n In [It] (Node Runtime: 1h29m59.877s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42\n At [By Step] getting admin credentials for the cluster (Step Runtime: 5m38.233s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:157\n\n Spec Goroutine\n goroutine 87 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8cb0998, 0xc000a76850}, 0x81dda58?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cfd5c0, {0x8cb0998, 0xc000a76850}, 0xc000a449f0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetAdminRESTConfigForHCPCluster(0xc000b14640, {0x7fa0219e8860, 0xc000c710e0}, 0xc0000d9ec8, {0xc001810108, 0x17}, {0x81fbb46, 0x12}, 0x8bb2c97000)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:107\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func10.2({0x7fa0219e8860, 0xc000c710e0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:158\n | \n | By(\"getting admin credentials for the cluster\")\n \u003e adminRESTConfig, err := tc.GetAdminRESTConfigForHCPCluster(\n | \tctx,\n | \ttc.Get20240610ClientFactoryOrDie(ctx).NewHcpOpenShiftClustersClient(),\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc8ce0?, 0xc000c710e0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 40\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:165 @ 04/02/26 19:42:28.688\n \"ts\"=\"2026-04-02 19:42:28.689138\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 19:42:28.689210\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-private-kv-cluster-brmrkf\" \"resourceGroup\"=\"private-keyvault-w6qm7t\"\n \"ts\"=\"2026-04-02 19:42:28.802742\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-private-kv-cluster-brmrkf\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 19:42:29.228630\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-65e261cf588b9f2637c69902ef65386ae7e651e35485a39612d668d6a3309cde.yaml.gz\"\n \"ts\"=\"2026-04-02 19:42:29.228771\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-65e261cf588b9f2637c69902ef65386ae7e651e35485a39612d668d6a3309cde.yaml.gz\"\n \"ts\"=\"2026-04-02 19:42:29.228877\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 19:42:29.231553\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-18\"\n \"ts\"=\"2026-04-02 19:42:29.231567\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"private-keyvault-w6qm7t\"\n \"ts\"=\"2026-04-02 19:42:29.323497\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"private-keyvault-w6qm7t\" \"deployment\"=\"pooledNonMsiScopedAssignments-had2isw6vkgre\"\n \"ts\"=\"2026-04-02 19:42:29.375537\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-18\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 19:42:29.376346\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"private-keyvault-w6qm7t\" \"deployment\"=\"customer-infra-private-kv-cluster-brmrkf\"\n \"ts\"=\"2026-04-02 19:42:29.521817\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-18\" \"deployment\"=\"msi-deployment-18\"\n \"ts\"=\"2026-04-02 19:42:29.680009\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 19:42:29.680123\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 19:42:29.680252\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"private-keyvault-w6qm7t\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"c568dfc23f9463cada9f64325346775fa1b4ed7ec34213db058b2a6d0d5a4406","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","failed_at":"e2e","signature_id":"e39559634ab284bff6218016b0a95f285d7a8af2e09376008215d96e16037f4e","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010281e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-subnet-reuse-fnhchx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010281a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001028180\u003e{\n msg: \"failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0006421b0\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 18:13:11 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n \"X-Ms-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"115\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010281e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-subnet-reuse-fnhchx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010281a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001028180\u003e{\n msg: \"failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0006421b0\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 18:13:11 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n \"X-Ms-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"115\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d78c0bca89c95459375355a5923414dd1603fcc7a1039f0264667a1b20c77dc3","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}},{"row_id":"f78155096188460041e3ccff70b520942eda88f0ffd28a62e51dd54ff3b5c2b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["d78c0bca89c95459375355a5923414dd1603fcc7a1039f0264667a1b20c77dc3","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T16:29:01Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9d2f81e5ccfddd007f3970c2e0e23e74f4ee4433b9f068c226063dea74b240d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","occurred_at":"2026-04-02T16:29:01Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"f72d87a1d0052c0ffe0dc428b6905cb2893fdad01d7edfdfdf84c2fc2a76eea7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","occurred_at":"2026-04-02T16:29:01Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734985123958784","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T16:01:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T16:00:42Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a5010aa18525f9d5702d669fdb9fcd486cefddc7d6432a7e0788bc51af783072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","occurred_at":"2026-04-02T16:00:42Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"c545ae7e86fc8a6a7a6087f509dee84bc68e916ba7065aa352ce3fc0e82efeda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","occurred_at":"2026-04-02T16:00:42Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T15:56:01Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"b7a3f1090d85b8d7c1741bd0cfa59b0e2d8eb172f4dd1bd2756f2de645d04f5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","occurred_at":"2026-04-02T15:56:01Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"ef7c9ce9651baa1148e78915cb37d729d91a87fa3da2fb43786413701ff59242","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","occurred_at":"2026-04-02T15:56:01Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039683369729527808","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T12:36:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T10:28:45Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"17dd165ab965e3115de0dc0782b2c133ef0ff108645a1ec14fa49c1536ce978f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"44515d65024530a8d836482aa9217a149ecc247693a591f8f4dee27fb8922f6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"cacb611a734fddabffd52719f6c97cb7e2d4671c63fe6ec26bfa95241c774c1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"d7cebf0cce6c045530e4dc5473b13ed9ff2ddf137535eb60f0993b38437fb0b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T00:18:47Z"},"failed_at":["e2e","unknown"],"failed_test_count":30,"occurrences":[{"row_id":"0798106f5b7bccc6e1292d97a74fd73ea146a6c1cfddfc22694705b96012d406","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"c4518cf266a9ae8b9147a4bc9108884198c498a4e0ee55b158614588b552026d","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093407 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:67\n STEP: creating a resource group @ 04/02/26 01:30:11.709\n \"ts\"=\"2026-04-02 01:30:11.709287\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n STEP: creating cluster parameters @ 04/02/26 01:30:15.537\n \"ts\"=\"2026-04-02 01:30:15.537817\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:30:15.537832\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources (infrastructure and managed identities) @ 04/02/26 01:30:15.537\n STEP: creating the HCP cluster @ 04/02/26 01:31:11.592\n \"ts\"=\"2026-04-02 01:31:11.592927\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ephemeral-disk\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112 @ 04/02/26 02:16:11.595\n \"ts\"=\"2026-04-02 02:16:11.596087\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:16:11.596281\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ephemeral-disk-6gkhgv\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n \"ts\"=\"2026-04-02 02:16:11.767154\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ephemeral-disk-6gkhgv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:16:13.253101\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:13.253248\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:13.253328\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:13.255519\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n \"ts\"=\"2026-04-02 02:16:13.255507\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-128\"\n \"ts\"=\"2026-04-02 02:16:13.391863\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\" \"deployment\"=\"pooledNonMsiScopedAssignments-6xts4ycasve7q\"\n \"ts\"=\"2026-04-02 02:16:13.480661\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\" \"deployment\"=\"customer-infra-ephemeral-disk-6gkhgv\"\n \"ts\"=\"2026-04-02 02:16:13.690527\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-128\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:16:13.855407\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-128\" \"deployment\"=\"msi-deployment-128\"\n \"ts\"=\"2026-04-02 02:16:13.997026\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:13.997138\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:16:13.997296\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093407 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:67\n STEP: creating a resource group @ 04/02/26 01:30:11.709\n \"ts\"=\"2026-04-02 01:30:11.709287\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n STEP: creating cluster parameters @ 04/02/26 01:30:15.537\n \"ts\"=\"2026-04-02 01:30:15.537817\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:30:15.537832\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources (infrastructure and managed identities) @ 04/02/26 01:30:15.537\n STEP: creating the HCP cluster @ 04/02/26 01:31:11.592\n \"ts\"=\"2026-04-02 01:31:11.592927\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ephemeral-disk\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112 @ 04/02/26 02:16:11.595\n \"ts\"=\"2026-04-02 02:16:11.596087\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:16:11.596281\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ephemeral-disk-6gkhgv\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n \"ts\"=\"2026-04-02 02:16:11.767154\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ephemeral-disk-6gkhgv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:16:13.253101\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:13.253248\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:13.253328\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:13.255519\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\n \"ts\"=\"2026-04-02 02:16:13.255507\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-128\"\n \"ts\"=\"2026-04-02 02:16:13.391863\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\" \"deployment\"=\"pooledNonMsiScopedAssignments-6xts4ycasve7q\"\n \"ts\"=\"2026-04-02 02:16:13.480661\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\" \"deployment\"=\"customer-infra-ephemeral-disk-6gkhgv\"\n \"ts\"=\"2026-04-02 02:16:13.690527\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-128\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:16:13.855407\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-128\" \"deployment\"=\"msi-deployment-128\"\n \"ts\"=\"2026-04-02 02:16:13.997026\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:13.997138\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:16:13.997296\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ephemeral-osdisk-74r57q\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"107d597d18ea98ba7911030605080ce287b63964ea647f421472e456c81bae3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"20cc6f67b762823d979736e097ec3b236ea3bc2eaa4a0f6f645004250bc4c285","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\"np-version-upgrade-cluster-z8jp8j\" in resourcegroup=\"rg-np-version-upgrade-z8jp8j-brqz27\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"name\": \"3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:33:37.647438148Z\",\n \"endTime\": \"2026-04-02T01:34:08.890577638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea7c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\"np-version-upgrade-cluster-z8jp8j\" in resourcegroup=\"rg-np-version-upgrade-z8jp8j-brqz27\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"name\": \"3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:33:37.647438148Z\",\n \"endTime\": \"2026-04-02T01:34:08.890577638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea7c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3faf68220dd7536a7ab4746fe0c4241f0ba52074a7b568e4e5cd42f42f541cba","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16daa9098921a1a2d318d999d9e7e0d7e106511140073db232b376b794fdf1ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"d77a128e9ccb342aa703c1ffc152a70ad5d80aaab8aaba6ca7703762895ebdcf","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011c08a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-dhfb8w\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-dhfb8w, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-dhfb8w\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-dhfb8w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011434e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-dhfb8w\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011c08a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-dhfb8w\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-dhfb8w, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-dhfb8w\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-dhfb8w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011434e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-dhfb8w\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"17f47e71446bf30deb1d4479fdae03e96185d96335d20e61a4c4c2d51ba8c454","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"544adc8e9ab97df3076f3902b260c61ba87e23a109d40f5068d246ae8edf5702","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093608 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/02/26 01:33:28.529\n \"ts\"=\"2026-04-02 01:33:28.529807\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n STEP: creating cluster parameters @ 04/02/26 01:33:32.008\n \"ts\"=\"2026-04-02 01:33:32.008858\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:33:32.008877\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:33:32.008\n STEP: creating the HCP cluster @ 04/02/26 01:34:28.975\n \"ts\"=\"2026-04-02 01:34:28.975113\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:117 @ 04/02/26 02:19:28.976\n \"ts\"=\"2026-04-02 02:19:28.976618\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:19:28.976714\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-w42dxb\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n \"ts\"=\"2026-04-02 02:19:29.118328\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-w42dxb\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:19:30.642103\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-02 02:19:30.642232\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-02 02:19:30.642302\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:19:30.643746\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-91\"\n \"ts\"=\"2026-04-02 02:19:30.643775\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n \"ts\"=\"2026-04-02 02:19:30.801321\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-q25j97\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-04-02 02:19:30.816599\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-91\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:19:30.874154\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-q25j97\" \"deployment\"=\"customer-infra-negative-tests-cluster-w42dxb\"\n \"ts\"=\"2026-04-02 02:19:30.957443\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-91\" \"deployment\"=\"msi-deployment-91\"\n \"ts\"=\"2026-04-02 02:19:31.103536\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:19:31.103654\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:19:31.103784\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093608 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/02/26 01:33:28.529\n \"ts\"=\"2026-04-02 01:33:28.529807\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n STEP: creating cluster parameters @ 04/02/26 01:33:32.008\n \"ts\"=\"2026-04-02 01:33:32.008858\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:33:32.008877\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:33:32.008\n STEP: creating the HCP cluster @ 04/02/26 01:34:28.975\n \"ts\"=\"2026-04-02 01:34:28.975113\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:117 @ 04/02/26 02:19:28.976\n \"ts\"=\"2026-04-02 02:19:28.976618\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:19:28.976714\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-w42dxb\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n \"ts\"=\"2026-04-02 02:19:29.118328\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-w42dxb\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:19:30.642103\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-02 02:19:30.642232\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-02 02:19:30.642302\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:19:30.643746\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-91\"\n \"ts\"=\"2026-04-02 02:19:30.643775\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\n \"ts\"=\"2026-04-02 02:19:30.801321\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-q25j97\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-04-02 02:19:30.816599\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-91\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:19:30.874154\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-q25j97\" \"deployment\"=\"customer-infra-negative-tests-cluster-w42dxb\"\n \"ts\"=\"2026-04-02 02:19:30.957443\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-91\" \"deployment\"=\"msi-deployment-91\"\n \"ts\"=\"2026-04-02 02:19:31.103536\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:19:31.103654\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:19:31.103784\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-q25j97\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"31411b87ee725f0b21753350f38c826a369d93fbfcdb4f03ac74e7d9a3cb15b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"e63c24cd1d8891d1ce96daa3f00c9145ffedccd612aa864496c0aca698a66afe","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff1020\u003e: \n failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-txvwqn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-txvwqn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ccfdd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn\",\n },\n \u003c*fmt.wrapErrors | 0xc000ff0ff0\u003e{\n msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-txvwqn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ccfdd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff1020\u003e: \n failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-txvwqn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-txvwqn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ccfdd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn\",\n },\n \u003c*fmt.wrapErrors | 0xc000ff0ff0\u003e{\n msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-txvwqn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ccfdd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-txvwqn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"31a009c6f88e946d3ed84080f07c1d82787c8da06debfd880cc83da9935346f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-02T02:21:49Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"361ee1dd5047b9e3a80a2c7c7ca1732c3773dde99b436c0a5665a8e3c6413329","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"bcb22337065c03471ff7a0d8c4db52818ec22b275d83d1ee92f21bc5c25bc706","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000894f20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8sxmq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"name\": \"a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:22:54.935266426Z\",\n \"endTime\": \"2026-04-02T00:23:25.288802016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000894f00\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authori...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000894f20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8sxmq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"name\": \"a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:22:54.935266426Z\",\n \"endTime\": \"2026-04-02T00:23:25.288802016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000894f00\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authori...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c36ae336b29ede965426348568977f4287cdb5ff3cdeab9c4ba7530a7df03c1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"482828eaa90e7e7da467c17b0ef30f2a2ad89584585bc4d4e252d424ab17d25b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"6d289eb97cdf8c771abc718ea433d15081de7b5463711d7aa5396be1ace3c0a4","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0017000c0\u003e: \n failed to create HCP cluster ea-listrvdprm, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: failed waiting for cluster=\"ea-listrvdprm\" in resourcegroup=\"ea-list-7cqq2d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listrvdprm, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: failed waiting for cluster=\\\"ea-listrvdprm\\\" in resourcegroup=\\\"ea-list-7cqq2d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000b04540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d\",\n },\n \u003c*fmt.wrapErrors | 0xc001700090\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listrvdprm\\\" in resourcegroup=\\\"ea-list-7cqq2d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000b04540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0017000c0\u003e: \n failed to create HCP cluster ea-listrvdprm, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: failed waiting for cluster=\"ea-listrvdprm\" in resourcegroup=\"ea-list-7cqq2d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listrvdprm, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: failed waiting for cluster=\\\"ea-listrvdprm\\\" in resourcegroup=\\\"ea-list-7cqq2d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000b04540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d\",\n },\n \u003c*fmt.wrapErrors | 0xc001700090\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listrvdprm\\\" in resourcegroup=\\\"ea-list-7cqq2d\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000b04540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listrvdprm in resource group ea-list-7cqq2d\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"4a685130d8ed5bb0c8cd62d5adec26655a28ce78ded1cc702729c642e08a6eee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"88e82f944c139e6cc19b557b3a082418af51be333b6988578d68f6c17833d85a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e0d50\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9sfb27\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9sfb27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00119cac0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27\",\n },\n \u003c*fmt.wrapErrors | 0xc0011e0d20\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9sfb27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00119cac0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e0d50\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9sfb27\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9sfb27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00119cac0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27\",\n },\n \u003c*fmt.wrapErrors | 0xc0011e0d20\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-9sfb27\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00119cac0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-9sfb27\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"4e51f0922e588e7b1f1c3d4b951c2bab7c598bf1ade3ea47bd812dae59aa699a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"8eb97e22943903cb8d934c6e5984a6ccdef7fd32962faac882fba587161198b5","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00078d420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-p4zvhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"name\": \"f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:27:03.073725945Z\",\n \"endTime\": \"2026-04-02T00:27:41.510479828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00078d3c0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not ha...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00078d420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-p4zvhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"name\": \"f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:27:03.073725945Z\",\n \"endTime\": \"2026-04-02T00:27:41.510479828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00078d3c0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not ha...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c36ae336b29ede965426348568977f4287cdb5ff3cdeab9c4ba7530a7df03c1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"65d757c59df62d186d2ca1b221c5f76abb0bacb5a1c90b9736690aba41b81074","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"343b69b9abdb0dbbbe18730a094b99c94da5fe39b0bfe0bc8d50516f80da0fc6","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"\u003e github.com/Azure/ARO-HCP/test/e2e.init.func17.2({0x7f395865ae60, 0xc000a84360})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc2c80?, 0xc000a84360?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 73\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 04/02/26 01:51:54.509\n \"ts\"=\"2026-04-02 01:51:54.509749\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.509817\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-68kpl4\" \"resourceGroup\"=\"customer-rg-wz4brf\"\n \"ts\"=\"2026-04-02 01:51:54.669907\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-68kpl4\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:55.462809\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-nw72jh\" \"resourceGroup\"=\"customer-rg-wz4brf\"\n \"ts\"=\"2026-04-02 01:51:55.588392\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-nw72jh\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:57.012940\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.013048\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.013105\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:57.014271\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-109\"\n \"ts\"=\"2026-04-02 01:51:57.014292\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\"\n \"ts\"=\"2026-04-02 01:51:57.014281\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-113\"\n \"ts\"=\"2026-04-02 01:51:57.014340\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-wz4brf\"\n \"ts\"=\"2026-04-02 01:51:57.247683\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-04-02 01:51:57.326543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-nw72jh\"\n \"ts\"=\"2026-04-02 01:51:57.361557\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-113\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.433058\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-04-02 01:51:57.501750\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-113\" \"deployment\"=\"msi-deployment-113\"\n \"ts\"=\"2026-04-02 01:51:57.509668\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"customer-infra-basic-hcp-cluster-68kpl4\"\n \"ts\"=\"2026-04-02 01:51:57.534043\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.557500\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-109\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.665194\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"msi-deployment-123\"\n \"ts\"=\"2026-04-02 01:51:57.740187\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-109\" \"deployment\"=\"msi-deployment-109\"\n \"ts\"=\"2026-04-02 01:51:57.874853\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:57.874990\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:57.875105\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-wz4brf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func17.2({0x7f395865ae60, 0xc000a84360})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc2c80?, 0xc000a84360?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 73\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 04/02/26 01:51:54.509\n \"ts\"=\"2026-04-02 01:51:54.509749\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.509817\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-68kpl4\" \"resourceGroup\"=\"customer-rg-wz4brf\"\n \"ts\"=\"2026-04-02 01:51:54.669907\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-68kpl4\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:55.462809\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-nw72jh\" \"resourceGroup\"=\"customer-rg-wz4brf\"\n \"ts\"=\"2026-04-02 01:51:55.588392\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-nw72jh\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:57.012940\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.013048\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.013105\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:57.014271\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-109\"\n \"ts\"=\"2026-04-02 01:51:57.014292\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\"\n \"ts\"=\"2026-04-02 01:51:57.014281\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-113\"\n \"ts\"=\"2026-04-02 01:51:57.014340\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-wz4brf\"\n \"ts\"=\"2026-04-02 01:51:57.247683\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-04-02 01:51:57.326543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-nw72jh\"\n \"ts\"=\"2026-04-02 01:51:57.361557\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-113\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.433058\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-04-02 01:51:57.501750\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-113\" \"deployment\"=\"msi-deployment-113\"\n \"ts\"=\"2026-04-02 01:51:57.509668\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-wz4brf\" \"deployment\"=\"customer-infra-basic-hcp-cluster-68kpl4\"\n \"ts\"=\"2026-04-02 01:51:57.534043\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.557500\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-109\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.665194\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"msi-deployment-123\"\n \"ts\"=\"2026-04-02 01:51:57.740187\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-109\" \"deployment\"=\"msi-deployment-109\"\n \"ts\"=\"2026-04-02 01:51:57.874853\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:57.874990\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:57.875105\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-wz4brf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"6b8117a8efa98f67da71fb07eb58cf1fd87a58341fc88ad0f0faa43086c4177b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"760e7b83d354480bf49188911ed5c051f4b34e00e2943fc1a4213892da050339","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010502c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8xlw7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"name\": \"3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:29:14.718319727Z\",\n \"endTime\": \"2026-04-02T01:30:18.264779861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010502a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b5319333...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010502c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8xlw7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"name\": \"3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:29:14.718319727Z\",\n \"endTime\": \"2026-04-02T01:30:18.264779861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010502a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b5319333...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c36ae336b29ede965426348568977f4287cdb5ff3cdeab9c4ba7530a7df03c1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86f3b4210a7fbe1f68d88d9e200db601e6425e99cbb2e228eb3606b429382281","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"10a831987a4be893a9ce87869d689cc985a10c8e1bb02e94c90780c844b62b5d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"prod/parallel","failure_text":"inkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000db0b98, {0xc000c64f60?, 0x0?, 0x0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000932a88, {0x0, 0x0}, {0xc1de280, 0x0, 0x0}, {0xc0000bc064, 0x2a}, 0x0, {0xc000c64f40, ...})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000bc064, 0x2a}, ...)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2d45a85?, 0xc000c00c60?})\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8caa8c8, 0xc000b5cff0}, 0xc000b44900, 0x1)\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230\n\n goroutine 99 [select]\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000b44900?)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 98\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137 @ 04/02/26 01:51:54.509\n \"ts\"=\"2026-04-02 01:51:54.510196\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.510248\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-zxltj9-ncfxtn\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\"\n \"ts\"=\"2026-04-02 01:51:55.897221\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-zstream-4-19-zxltj9-ncfxtn\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:58.433435\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:58.433576\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:58.433670\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:58.435153\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-100\"\n \"ts\"=\"2026-04-02 01:51:58.435190\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\"\n \"ts\"=\"2026-04-02 01:51:58.537030\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\" \"deployment\"=\"pooledNonMsiScopedAssignments-fldd5huuy6tgg\"\n \"ts\"=\"2026-04-02 01:51:58.617554\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-100\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:58.695091\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-zxltj9-ncfxtn\"\n \"ts\"=\"2026-04-02 01:51:58.752078\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-100\" \"deployment\"=\"msi-deployment-100\"\n \"ts\"=\"2026-04-02 01:51:58.906467\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:58.906589\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:58.906768\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ninkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000db0b98, {0xc000c64f60?, 0x0?, 0x0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000932a88, {0x0, 0x0}, {0xc1de280, 0x0, 0x0}, {0xc0000bc064, 0x2a}, 0x0, {0xc000c64f40, ...})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000bc064, 0x2a}, ...)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2d45a85?, 0xc000c00c60?})\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8caa8c8, 0xc000b5cff0}, 0xc000b44900, 0x1)\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230\n\n goroutine 99 [select]\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000b44900?)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 98\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137 @ 04/02/26 01:51:54.509\n \"ts\"=\"2026-04-02 01:51:54.510196\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.510248\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-zxltj9-ncfxtn\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\"\n \"ts\"=\"2026-04-02 01:51:55.897221\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-zstream-4-19-zxltj9-ncfxtn\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:58.433435\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:58.433576\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-0c1d721d40414cddab3a52b9d7add0203efdd55a4c816fc13c12379212f5cea0.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:58.433670\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:58.435153\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-100\"\n \"ts\"=\"2026-04-02 01:51:58.435190\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\"\n \"ts\"=\"2026-04-02 01:51:58.537030\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\" \"deployment\"=\"pooledNonMsiScopedAssignments-fldd5huuy6tgg\"\n \"ts\"=\"2026-04-02 01:51:58.617554\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-100\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:58.695091\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\" \"deployment\"=\"customer-infra-cluster-zstream-4-19-zxltj9-ncfxtn\"\n \"ts\"=\"2026-04-02 01:51:58.752078\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-100\" \"deployment\"=\"msi-deployment-100\"\n \"ts\"=\"2026-04-02 01:51:58.906467\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:58.906589\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:58.906768\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-zstream-upgrade-4-19-d4r4n7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"871e7292f4256cdfc0fc9eeee654aa77a61e15420012f53dc97344821de0abe2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"c6bc5c31aec45d33666384710b83a3f2ff5b4f5da157f7118eaf6285453023bf","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001533530\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pw2btq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-pw2btq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f77d30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq\",\n },\n \u003c*fmt.wrapErrors | 0xc001533500\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-pw2btq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f77d30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001533530\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pw2btq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-pw2btq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f77d30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq\",\n },\n \u003c*fmt.wrapErrors | 0xc001533500\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-pw2btq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f77d30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-pw2btq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"8839527dc47212fd81a34175bfef80f002dd6e6a4926f5f0f661ea3a76bbdddc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"8c013b50bbb0e285088843f7ca56a5d04d08e5252671dcc9fa791d1c881f1bb6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001372f90\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-wvwqlp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-wvwqlp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013bd230\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp\",\n },\n \u003c*fmt.wrapErrors | 0xc001372f60\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-wvwqlp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013bd230\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001372f90\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-wvwqlp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-wvwqlp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013bd230\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp\",\n },\n \u003c*fmt.wrapErrors | 0xc001372f60\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-wvwqlp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013bd230\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-wvwqlp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"896ee50002db939161bab8534b74711dbe5c51ded97ed49d97c8fd6a40c19bc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"bc84585441cba890625bb3af9f5f77ae221485dd55718ed23036a13e579acb7c","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001104e10\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-c4nrww\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-c4nrww\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015dd0e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww\",\n },\n \u003c*fmt.wrapErrors | 0xc001104de0\u003e{\n msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-c4nrww\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015dd0e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001104e10\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-c4nrww\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-c4nrww\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015dd0e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww\",\n },\n \u003c*fmt.wrapErrors | 0xc001104de0\u003e{\n msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-c4nrww\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015dd0e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-c4nrww\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"8a0ba5a399a6886939677a1bd5e085b71e4787d42a0efad93fe26992c6169cdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"879079e6ffdb78b0cfdd72cea6ac98ec70779604b0c6ae9831a74dc3d4e76e14","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"prod/parallel","failure_text":"inkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000d5cb98, {0xc000b0a9a0?, 0x0?, 0x0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000d3a008, {0x0, 0x0}, {0xc1de280, 0x0, 0x0}, {0xc0000bc064, 0x2a}, 0x0, {0xc000b0a980, ...})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000bc064, 0x2a}, ...)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2d45a85?, 0xc000a92c60?})\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8caa8c8, 0xc000913cc0}, 0xc001012f00, 0x1)\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230\n\n goroutine 88 [select]\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc001012f00?)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 87\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137 @ 04/02/26 01:51:54.631\n \"ts\"=\"2026-04-02 01:51:54.631696\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.631774\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-zstream-4-20-dcg5c9-xksvgn\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\"\n \"ts\"=\"2026-04-02 01:51:55.940564\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-zstream-4-20-dcg5c9-xksvgn\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:57.236284\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-c1cc868701677ba9bad3c4e2620f4c229f47803f50af63ee5732ea435a94776c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.236400\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-c1cc868701677ba9bad3c4e2620f4c229f47803f50af63ee5732ea435a94776c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.236476\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:57.237670\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-142\"\n \"ts\"=\"2026-04-02 01:51:57.237687\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\"\n \"ts\"=\"2026-04-02 01:51:57.399756\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-142\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.542142\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-142\" \"deployment\"=\"msi-deployment-142\"\n \"ts\"=\"2026-04-02 01:51:57.799719\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\" \"deployment\"=\"pooledNonMsiScopedAssignments-ufgsuor5fm6ve\"\n \"ts\"=\"2026-04-02 01:51:58.700077\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\" \"deployment\"=\"customer-infra-cluster-zstream-4-20-dcg5c9-xksvgn\"\n \"ts\"=\"2026-04-02 01:51:59.922401\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:59.922536\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:59.922704\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ninkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000d5cb98, {0xc000b0a9a0?, 0x0?, 0x0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000d3a008, {0x0, 0x0}, {0xc1de280, 0x0, 0x0}, {0xc0000bc064, 0x2a}, 0x0, {0xc000b0a980, ...})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000bc064, 0x2a}, ...)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2d45a85?, 0xc000a92c60?})\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/ginkgo/util.go:85\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8caa8c8, 0xc000913cc0}, 0xc001012f00, 0x1)\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:596\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:237\n github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230\n\n goroutine 88 [select]\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc001012f00?)\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 87\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137 @ 04/02/26 01:51:54.631\n \"ts\"=\"2026-04-02 01:51:54.631696\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.631774\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-zstream-4-20-dcg5c9-xksvgn\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\"\n \"ts\"=\"2026-04-02 01:51:55.940564\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-zstream-4-20-dcg5c9-xksvgn\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:57.236284\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-c1cc868701677ba9bad3c4e2620f4c229f47803f50af63ee5732ea435a94776c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.236400\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-c1cc868701677ba9bad3c4e2620f4c229f47803f50af63ee5732ea435a94776c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:57.236476\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:57.237670\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-142\"\n \"ts\"=\"2026-04-02 01:51:57.237687\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\"\n \"ts\"=\"2026-04-02 01:51:57.399756\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-142\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:57.542142\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-142\" \"deployment\"=\"msi-deployment-142\"\n \"ts\"=\"2026-04-02 01:51:57.799719\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\" \"deployment\"=\"pooledNonMsiScopedAssignments-ufgsuor5fm6ve\"\n \"ts\"=\"2026-04-02 01:51:58.700077\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\" \"deployment\"=\"customer-infra-cluster-zstream-4-20-dcg5c9-xksvgn\"\n \"ts\"=\"2026-04-02 01:51:59.922401\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:59.922536\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:59.922704\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-zstream-upgrade-4-20-zcjg2t\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"8e057810565a578d36095cf68bd80552d8dc9a7f7f3f82ba340b9664a404f2cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"f5b50909a454a74f214ea8348c5bbbdd39c94bd8e1438791df15454e3358b666","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00131a180\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\"cluster-zstream-4-21-xvw7fn\" in resourcegroup=\"rg-zstream-upgrade-4-21-qf98rq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"name\": \"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:34:57.397134314Z\",\n \"endTime\": \"2026-04-02T01:36:07.715905327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00131a160\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00131a180\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\"cluster-zstream-4-21-xvw7fn\" in resourcegroup=\"rg-zstream-upgrade-4-21-qf98rq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"name\": \"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:34:57.397134314Z\",\n \"endTime\": \"2026-04-02T01:36:07.715905327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00131a160\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b16feefd927dd52de60b3d699308db2fee6e43cd349d94e0653b4b9fce6f419","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a616166ae07c110a53ff85011a0fd814fcc9ad6a9d56ec3fea201e1d2587749","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"d284e527b2816b754418e6bc1461dca78d18ee8332b2f10fc25ac8eb7a7b7695","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093289 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 04/02/26 01:28:12.111\n \"ts\"=\"2026-04-02 01:28:12.111559\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n STEP: creating cluster parameters @ 04/02/26 01:28:15.958\n \"ts\"=\"2026-04-02 01:28:15.958468\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:28:15.958480\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:28:15.958\n STEP: creating the HCP cluster @ 04/02/26 01:29:12.294\n \"ts\"=\"2026-04-02 01:29:12.294519\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83 @ 04/02/26 02:14:12.297\n \"ts\"=\"2026-04-02 02:14:12.297853\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:14:12.297926\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-bpqwfl\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n \"ts\"=\"2026-04-02 02:14:12.443280\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-bpqwfl\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:14:13.765625\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:14:13.765760\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:14:13.765865\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:14:13.767015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-140\"\n \"ts\"=\"2026-04-02 02:14:13.767036\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n \"ts\"=\"2026-04-02 02:14:13.873433\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-04-02 02:14:13.928981\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-140\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:14:13.948021\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\" \"deployment\"=\"customer-infra-np-autoscale-cluster-bpqwfl\"\n \"ts\"=\"2026-04-02 02:14:14.080153\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-140\" \"deployment\"=\"msi-deployment-140\"\n \"ts\"=\"2026-04-02 02:14:14.225391\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:14:14.225510\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:14:14.225635\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093289 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 04/02/26 01:28:12.111\n \"ts\"=\"2026-04-02 01:28:12.111559\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n STEP: creating cluster parameters @ 04/02/26 01:28:15.958\n \"ts\"=\"2026-04-02 01:28:15.958468\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:28:15.958480\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:28:15.958\n STEP: creating the HCP cluster @ 04/02/26 01:29:12.294\n \"ts\"=\"2026-04-02 01:29:12.294519\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83 @ 04/02/26 02:14:12.297\n \"ts\"=\"2026-04-02 02:14:12.297853\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:14:12.297926\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-bpqwfl\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n \"ts\"=\"2026-04-02 02:14:12.443280\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-bpqwfl\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:14:13.765625\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:14:13.765760\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:14:13.765865\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:14:13.767015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-140\"\n \"ts\"=\"2026-04-02 02:14:13.767036\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\n \"ts\"=\"2026-04-02 02:14:13.873433\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-04-02 02:14:13.928981\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-140\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:14:13.948021\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\" \"deployment\"=\"customer-infra-np-autoscale-cluster-bpqwfl\"\n \"ts\"=\"2026-04-02 02:14:14.080153\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-140\" \"deployment\"=\"msi-deployment-140\"\n \"ts\"=\"2026-04-02 02:14:14.225391\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:14:14.225510\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:14:14.225635\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-wxdqdk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"b0004f238527dde869f80a0e9cdbfe0e2139bd2b5f00134db38063204bbc37a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"28ca8e03fd5320e3f8efb638a095e3b0dd77134ad11a7b771536a46c0242eb16","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d6660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mxj67k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-mxj67k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcc6d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d6660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mxj67k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-mxj67k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcc6d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"b57a1b4c83cfccda568abdd6d7e3399a2e46a50a41f7f16296a4e27ffb01752a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"bf7bbb8a1a3c60164c716ce2408d462e0ad38d9be27c4c85517643e4d37fa8d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"0712c50b4b58d80c7fa67f9c8b147184116e65cd92f9e399e67019ad4489ec1a","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c8c90\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003c*fmt.wrapErrors | 0xc0009c8c60\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c8c90\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003c*fmt.wrapErrors | 0xc0009c8c60\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"d207a043c9e729edbd3b6607645958aba42abb1d8f04a3a949f02a7844b02ab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"f71f8c82e2c81fb9ce9bd2b9c751d34e51f6f04d77b8d2ae0b9f4bedf99710eb","test_name":"Customer should be able to create an HCP cluster","test_suite":"prod/parallel","failure_text":"ining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to create an HCP cluster (Spec Runtime: 1h29m59.852s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n In [It] (Node Runtime: 1h29m59.852s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n At [By Step] creating the HCP cluster (Step Runtime: 15m5.938s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:88\n\n Spec Goroutine\n goroutine 81 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8caa8c8, 0xc001586820}, 0x81d9580?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8ce5ea0, {0x8caa8c8, 0xc001586820}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8caa938?, 0xc000704380?}, {{0x8cc2ba8?, 0xc000e26280?}, 0xc00093c648?}, 0xc0000d7768, {0xc00168e690, 0x14}, {0x81f1f5c, 0x11}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x81b78f8, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func18.2({0x7f585c2cf198, 0xc000e74e10})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:89\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc2c80?, 0xc000e74e10?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 78\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96 @ 04/02/26 01:51:54.509\n \"ts\"=\"2026-04-02 01:51:54.509787\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.509849\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qwz4qp\" \"resourceGroup\"=\"basic-cluster-448hmt\"\n \"ts\"=\"2026-04-02 01:51:54.642128\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-qwz4qp\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:55.916040\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:55.916147\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:55.916233\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:55.917374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-135\"\n \"ts\"=\"2026-04-02 01:51:55.917390\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"basic-cluster-448hmt\"\n \"ts\"=\"2026-04-02 01:51:56.033502\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-448hmt\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-04-02 01:51:56.066101\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-135\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:56.118598\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-448hmt\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qwz4qp\"\n \"ts\"=\"2026-04-02 01:51:56.406662\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-135\" \"deployment\"=\"msi-deployment-135\"\n \"ts\"=\"2026-04-02 01:51:56.541695\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:56.541805\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:56.541940\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"basic-cluster-448hmt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Customer should be able to create an HCP cluster (Spec Runtime: 1h29m59.852s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n In [It] (Node Runtime: 1h29m59.852s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n At [By Step] creating the HCP cluster (Step Runtime: 15m5.938s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:88\n\n Spec Goroutine\n goroutine 81 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8caa8c8, 0xc001586820}, 0x81d9580?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8ce5ea0, {0x8caa8c8, 0xc001586820}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8caa938?, 0xc000704380?}, {{0x8cc2ba8?, 0xc000e26280?}, 0xc00093c648?}, 0xc0000d7768, {0xc00168e690, 0x14}, {0x81f1f5c, 0x11}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x81b78f8, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func18.2({0x7f585c2cf198, 0xc000e74e10})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:89\n | \n | By(\"creating the HCP cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(\n | \tctx,\n | \tGinkgoLogr,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc2c80?, 0xc000e74e10?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 78\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96 @ 04/02/26 01:51:54.509\n \"ts\"=\"2026-04-02 01:51:54.509787\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.509849\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qwz4qp\" \"resourceGroup\"=\"basic-cluster-448hmt\"\n \"ts\"=\"2026-04-02 01:51:54.642128\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-qwz4qp\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:55.916040\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:55.916147\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:55.916233\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:55.917374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-135\"\n \"ts\"=\"2026-04-02 01:51:55.917390\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"basic-cluster-448hmt\"\n \"ts\"=\"2026-04-02 01:51:56.033502\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-448hmt\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-04-02 01:51:56.066101\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-135\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:56.118598\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-448hmt\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qwz4qp\"\n \"ts\"=\"2026-04-02 01:51:56.406662\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-135\" \"deployment\"=\"msi-deployment-135\"\n \"ts\"=\"2026-04-02 01:51:56.541695\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:56.541805\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:56.541940\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"basic-cluster-448hmt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e8f03a586fc6692a4ce41bd847d66fe2a903827c599fc7b168efa8ab01b28b7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"f57ec76b632f5772ac7a0f8849727df031d5ca56f807171233b950f03ff6df85","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093552 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to use workload identity via the cluster OIDC issuer URL [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n STEP: creating a resource group @ 04/02/26 01:32:32.392\n \"ts\"=\"2026-04-02 01:32:32.393025\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n STEP: creating cluster parameters @ 04/02/26 01:32:36.337\n \"ts\"=\"2026-04-02 01:32:36.337842\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:32:36.337858\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/02/26 01:32:36.337\n STEP: creating the HCP cluster @ 04/02/26 01:33:48.487\n \"ts\"=\"2026-04-02 01:33:48.487534\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216 @ 04/02/26 02:18:48.488\n \"ts\"=\"2026-04-02 02:18:48.488819\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:18:48.488889\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-oidc-wi-cluster-qgwwhq\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n \"ts\"=\"2026-04-02 02:18:48.616708\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-oidc-wi-cluster-qgwwhq\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:18:50.026272\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-02 02:18:50.026377\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-02 02:18:50.026457\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:18:50.027629\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\"\n \"ts\"=\"2026-04-02 02:18:50.027651\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n \"ts\"=\"2026-04-02 02:18:50.203926\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-hwdf78\" \"deployment\"=\"pooledNonMsiScopedAssignments-s6kwp7xg43dbi\"\n \"ts\"=\"2026-04-02 02:18:50.292891\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-hwdf78\" \"deployment\"=\"customer-infra-oidc-wi-cluster-qgwwhq\"\n \"ts\"=\"2026-04-02 02:18:50.386139\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:18:50.540807\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"msi-deployment-90\"\n \"ts\"=\"2026-04-02 02:18:50.676425\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:18:50.676568\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:18:50.676702\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093552 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to use workload identity via the cluster OIDC issuer URL [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n STEP: creating a resource group @ 04/02/26 01:32:32.392\n \"ts\"=\"2026-04-02 01:32:32.393025\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n STEP: creating cluster parameters @ 04/02/26 01:32:36.337\n \"ts\"=\"2026-04-02 01:32:36.337842\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:32:36.337858\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/02/26 01:32:36.337\n STEP: creating the HCP cluster @ 04/02/26 01:33:48.487\n \"ts\"=\"2026-04-02 01:33:48.487534\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216 @ 04/02/26 02:18:48.488\n \"ts\"=\"2026-04-02 02:18:48.488819\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:18:48.488889\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-oidc-wi-cluster-qgwwhq\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n \"ts\"=\"2026-04-02 02:18:48.616708\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-oidc-wi-cluster-qgwwhq\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:18:50.026272\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-02 02:18:50.026377\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-02 02:18:50.026457\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:18:50.027629\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\"\n \"ts\"=\"2026-04-02 02:18:50.027651\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\n \"ts\"=\"2026-04-02 02:18:50.203926\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-hwdf78\" \"deployment\"=\"pooledNonMsiScopedAssignments-s6kwp7xg43dbi\"\n \"ts\"=\"2026-04-02 02:18:50.292891\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-hwdf78\" \"deployment\"=\"customer-infra-oidc-wi-cluster-qgwwhq\"\n \"ts\"=\"2026-04-02 02:18:50.386139\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:18:50.540807\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"msi-deployment-90\"\n \"ts\"=\"2026-04-02 02:18:50.676425\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:18:50.676568\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:18:50.676702\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"oidc-wi-hwdf78\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"ea46a45cdea57f1d10a0579292f64d149c3086d55b6f1178e029721e14cbb81b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"5b24d397eafd651742199dea48c22fa110f454db09b76c3e8ee7360cd5472994","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f32720\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n2k8wb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n2k8wb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e8e3a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb\",\n },\n \u003c*fmt.wrapErrors | 0xc000f326f0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n2k8wb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e8e3a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f32720\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n2k8wb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n2k8wb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e8e3a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb\",\n },\n \u003c*fmt.wrapErrors | 0xc000f326f0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n2k8wb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e8e3a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n2k8wb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"ec4475b5df3f215e4a63f05e1639ace02db76ff8a4a50681605f2adff50e1343","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"b4fe70556501746681ecb3c63407563373fd3957d8ba80d71df1e8acfa098cf1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"nup.\n Here's a current progress report:\n Customer should be able to create an HCP cluster and manage pull secrets (Spec Runtime: 1h29m59.871s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:68\n In [It] (Node Runtime: 1h29m59.871s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:68\n At [By Step] Creating the cluster (Step Runtime: 13m55.176s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:126\n\n Spec Goroutine\n goroutine 106 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8caa8c8, 0xc0009b6370}, 0x81d9580?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8ce5ea0, {0x8caa8c8, 0xc0009b6370}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8caa938?, 0xc0006fe770?}, {{0x8cc2ba8?, 0xc0002fc000?}, 0xc0015b39f0?}, 0xc0000d7388, {0xc001462f00, 0x16}, {0x820df8d, 0x16}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x81b78f8, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func13.2({0x7fe88dfb0c60, 0xc00094f710})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:127\n | \n | By(\"Creating the cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(ctx,\n | \tGinkgoLogr,\n | \t*resourceGroup.Name,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc2c80?, 0xc00094f710?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133 @ 04/02/26 01:51:54.529\n \"ts\"=\"2026-04-02 01:51:54.529471\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.529531\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-lx649m\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\"\n \"ts\"=\"2026-04-02 01:51:54.696878\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-pullsecret-hcp-cluster-lx649m\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:56.149433\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:56.149565\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:56.149650\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:56.150896\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\"\n \"ts\"=\"2026-04-02 01:51:56.150922\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\"\n \"ts\"=\"2026-04-02 01:51:56.288868\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\" \"deployment\"=\"pooledNonMsiScopedAssignments-p2h32vyw2l2ma\"\n \"ts\"=\"2026-04-02 01:51:56.420385\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-lx649m\"\n \"ts\"=\"2026-04-02 01:51:56.704432\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:56.829125\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"msi-deployment-144\"\n \"ts\"=\"2026-04-02 01:51:56.986545\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:56.986670\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:56.986793\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nnup.\n Here's a current progress report:\n Customer should be able to create an HCP cluster and manage pull secrets (Spec Runtime: 1h29m59.871s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:68\n In [It] (Node Runtime: 1h29m59.871s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:68\n At [By Step] Creating the cluster (Step Runtime: 13m55.176s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:126\n\n Spec Goroutine\n goroutine 106 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8caa8c8, 0xc0009b6370}, 0x81d9580?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8ce5ea0, {0x8caa8c8, 0xc0009b6370}, 0x0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8caa938?, 0xc0006fe770?}, {{0x8cc2ba8?, 0xc0002fc000?}, 0xc0015b39f0?}, 0xc0000d7388, {0xc001462f00, 0x16}, {0x820df8d, 0x16}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694\n github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x81b78f8, 0x4}, ...}, ...)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func13.2({0x7fe88dfb0c60, 0xc00094f710})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:127\n | \n | By(\"Creating the cluster\")\n \u003e err = tc.CreateHCPClusterFromParam(ctx,\n | \tGinkgoLogr,\n | \t*resourceGroup.Name,\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc2c80?, 0xc00094f710?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133 @ 04/02/26 01:51:54.529\n \"ts\"=\"2026-04-02 01:51:54.529471\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 01:51:54.529531\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-lx649m\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\"\n \"ts\"=\"2026-04-02 01:51:54.696878\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-pullsecret-hcp-cluster-lx649m\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 01:51:56.149433\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:56.149565\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a760a96f7a26fb2915b8600ab91b283094b2c3bf2e64d6f8222da667d302697b.yaml.gz\"\n \"ts\"=\"2026-04-02 01:51:56.149650\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:56.150896\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\"\n \"ts\"=\"2026-04-02 01:51:56.150922\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\"\n \"ts\"=\"2026-04-02 01:51:56.288868\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\" \"deployment\"=\"pooledNonMsiScopedAssignments-p2h32vyw2l2ma\"\n \"ts\"=\"2026-04-02 01:51:56.420385\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\" \"deployment\"=\"customer-infra-pullsecret-hcp-cluster-lx649m\"\n \"ts\"=\"2026-04-02 01:51:56.704432\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 01:51:56.829125\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"msi-deployment-144\"\n \"ts\"=\"2026-04-02 01:51:56.986545\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 01:51:56.986670\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 01:51:56.986793\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"pullsecret-test-9fw4wg\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"f3a464f80cf9051d8e00a84856accd3ca103240a7d378a747eef9bf38f9febfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"5f61202956ac87e3d93f10c8a415a8d113dfe2e5738870debb3c16c7d7ea925d","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093416 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\n \"ts\"=\"2026-04-02 01:30:16.223704\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating resource group @ 04/02/26 01:30:16.592\n \"ts\"=\"2026-04-02 01:30:16.592682\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n STEP: creating cluster parameters at control plane version @ 04/02/26 01:30:20.35\n \"ts\"=\"2026-04-02 01:30:20.350778\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:30:20.350792\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:30:20.35\n STEP: creating the HCP cluster with version 4.21 @ 04/02/26 01:31:15.662\n \"ts\"=\"2026-04-02 01:31:15.662325\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-2tcvzl\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132 @ 04/02/26 02:16:15.663\n \"ts\"=\"2026-04-02 02:16:15.663428\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:16:15.663517\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-2tcvzl-zk272b\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n \"ts\"=\"2026-04-02 02:16:16.034813\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-2tcvzl-zk272b\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:16:17.548875\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:17.548974\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:17.549030\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:17.550112\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-122\"\n \"ts\"=\"2026-04-02 02:16:17.550138\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n \"ts\"=\"2026-04-02 02:16:17.698808\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\" \"deployment\"=\"pooledNonMsiScopedAssignments-3rz66um7kjo3u\"\n \"ts\"=\"2026-04-02 02:16:17.791141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-2tcvzl-zk272b\"\n \"ts\"=\"2026-04-02 02:16:17.867337\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-122\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:16:18.022305\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-122\" \"deployment\"=\"msi-deployment-122\"\n \"ts\"=\"2026-04-02 02:16:18.167675\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:18.167790\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:16:18.167929\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093416 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\n \"ts\"=\"2026-04-02 01:30:16.223704\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating resource group @ 04/02/26 01:30:16.592\n \"ts\"=\"2026-04-02 01:30:16.592682\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n STEP: creating cluster parameters at control plane version @ 04/02/26 01:30:20.35\n \"ts\"=\"2026-04-02 01:30:20.350778\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:30:20.350792\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:30:20.35\n STEP: creating the HCP cluster with version 4.21 @ 04/02/26 01:31:15.662\n \"ts\"=\"2026-04-02 01:31:15.662325\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-2tcvzl\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132 @ 04/02/26 02:16:15.663\n \"ts\"=\"2026-04-02 02:16:15.663428\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:16:15.663517\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-2tcvzl-zk272b\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n \"ts\"=\"2026-04-02 02:16:16.034813\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-2tcvzl-zk272b\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:16:17.548875\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:17.548974\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:17.549030\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:17.550112\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-122\"\n \"ts\"=\"2026-04-02 02:16:17.550138\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\n \"ts\"=\"2026-04-02 02:16:17.698808\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\" \"deployment\"=\"pooledNonMsiScopedAssignments-3rz66um7kjo3u\"\n \"ts\"=\"2026-04-02 02:16:17.791141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-2tcvzl-zk272b\"\n \"ts\"=\"2026-04-02 02:16:17.867337\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-122\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:16:18.022305\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-122\" \"deployment\"=\"msi-deployment-122\"\n \"ts\"=\"2026-04-02 02:16:18.167675\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:18.167790\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:16:18.167929\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-2tcvzl-6t9hp4\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"fbc637d13e37b1653beecec24e8df8af0c6b0899f29d8580d7d38857a831c7e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"a7cfda201a2b66ae296ad118419320731cc0627a8536ef129427b5ac3d420469","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009e3290\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6jth45\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6jth45, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6jth45\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6jth45, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcad80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6jth45\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009e3290\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6jth45\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6jth45, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-6jth45\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6jth45, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcad80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6jth45\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"fce31e662b5fb0f6946dde075fb934f897ccab39c69d8764af40d85e3155ece7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"0c557fcf018f9a61194a38b090f24f00dd3e276e1f2d7cfd13ebc026187bd706","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015f8f90\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j9ccbj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003c*fmt.wrapErrors | 0xc0015f8f60\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015f8f90\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j9ccbj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003c*fmt.wrapErrors | 0xc0015f8f60\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded"}},{"row_id":"ff0b4a0bed75f58de12b593655945dc96b436029e52ea75f3c88279e794e1bc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","failed_at":"e2e","signature_id":"c916d0a06bab1834b33b0882eec112cc16fb97ff309aae36491c5cdfa36ac14d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093414 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/02/26 01:30:16.183\n \"ts\"=\"2026-04-02 01:30:16.183334\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n STEP: creating cluster parameters @ 04/02/26 01:30:19.545\n \"ts\"=\"2026-04-02 01:30:19.545044\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:30:19.545057\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:30:19.545\n STEP: creating the HCP cluster @ 04/02/26 01:31:15.39\n \"ts\"=\"2026-04-02 01:31:15.390262\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91 @ 04/02/26 02:16:15.391\n \"ts\"=\"2026-04-02 02:16:15.391756\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:16:15.391835\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-cv7855\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n \"ts\"=\"2026-04-02 02:16:15.522013\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-cv7855\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:16:16.197911\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:16.198011\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:16.198082\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:16.199349\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-145\"\n \"ts\"=\"2026-04-02 02:16:16.199367\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n \"ts\"=\"2026-04-02 02:16:16.383278\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-145\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:16:16.517232\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-145\" \"deployment\"=\"msi-deployment-145\"\n \"ts\"=\"2026-04-02 02:16:16.652118\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-02 02:16:16.730191\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-cv7855\"\n \"ts\"=\"2026-04-02 02:16:17.286302\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:17.286461\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:16:17.286613\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775093414 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/02/26 01:30:16.183\n \"ts\"=\"2026-04-02 01:30:16.183334\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n STEP: creating cluster parameters @ 04/02/26 01:30:19.545\n \"ts\"=\"2026-04-02 01:30:19.545044\" \"level\"=0 \"msg\"=\"Using OpenShift control plane version\" \"version\"=\"4.20\"\n \"ts\"=\"2026-04-02 01:30:19.545057\" \"level\"=0 \"msg\"=\"Using OpenShift channel group\" \"channelGroup\"=\"stable\"\n STEP: creating customer resources @ 04/02/26 01:30:19.545\n STEP: creating the HCP cluster @ 04/02/26 01:31:15.39\n \"ts\"=\"2026-04-02 01:31:15.390262\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91 @ 04/02/26 02:16:15.391\n \"ts\"=\"2026-04-02 02:16:15.391756\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-02 02:16:15.391835\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-cv7855\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n \"ts\"=\"2026-04-02 02:16:15.522013\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-cv7855\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-02 02:16:16.197911\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:16.198011\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-02 02:16:16.198082\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:16.199349\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-145\"\n \"ts\"=\"2026-04-02 02:16:16.199367\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\n \"ts\"=\"2026-04-02 02:16:16.383278\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-145\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-02 02:16:16.517232\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-145\" \"deployment\"=\"msi-deployment-145\"\n \"ts\"=\"2026-04-02 02:16:16.652118\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-02 02:16:16.730191\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-cv7855\"\n \"ts\"=\"2026-04-02 02:16:17.286302\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-02 02:16:17.286461\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-02 02:16:17.286613\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-kd6zb5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}}],"failure_pattern_summary":{"signature_count":30,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","2b16feefd927dd52de60b3d699308db2fee6e43cd349d94e0653b4b9fce6f419","3faf68220dd7536a7ab4746fe0c4241f0ba52074a7b568e4e5cd42f42f541cba","c36ae336b29ede965426348568977f4287cdb5ff3cdeab9c4ba7530a7df03c1a","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":30,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}