{"meta":{"date":"2026-03-24","timezone":"UTC","generated_at":"2026-10-31T23:15:11Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":88,"failed_runs":40,"runs_with_occurrences":40,"runs_with_failure_pattern_matches":38,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":48,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036591336404029440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4614,"pr_state":"closed","pr_sha":"0ea6d9b25300bffbb91053c96049833941c1ccca","final_merged_sha":"0ea6d9b25300bffbb91053c96049833941c1ccca","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T23:49:32Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"28550d306a36b2d3db6d9bb0171f29409bc930d66564ec07020fd6b3d5b8fd6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036591336404029440","occurred_at":"2026-03-24T23:49:32Z","failed_at":"e2e","signature_id":"b29567ef051edc559289c78eba28826c4d090ad515e41c9a7ceae3bb84316409","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006966a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-rj49ln: failed starting cluster creation \"cluster-zstream-4-19-rj49ln\" in resourcegroup=\"rg-zstream-upgrade-4-19-pc2gx9\": PUT https://rp.j4029440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-pc2gx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-rj49ln\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-rj49ln: failed starting cluster creation \\\"cluster-zstream-4-19-rj49ln\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pc2gx9\\\": PUT https://rp.j4029440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-pc2gx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-rj49ln\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000696680\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-rj49ln\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pc2gx9\\\": PUT https://rp.j4029440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-pc2gx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-rj49ln\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00063aab0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"38dba7f1-fd4e-420c-a529-1198305aac8e\",\n ],\n \"Date\": [\n \"Wed, 25 Mar 2026 00:50:44 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"45\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0006965a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0006966a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-rj49ln: failed starting cluster creation \"cluster-zstream-4-19-rj49ln\" in resourcegroup=\"rg-zstream-upgrade-4-19-pc2gx9\": PUT https://rp.j4029440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-pc2gx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-rj49ln\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-rj49ln: failed starting cluster creation \\\"cluster-zstream-4-19-rj49ln\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pc2gx9\\\": PUT https://rp.j4029440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-pc2gx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-rj49ln\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000696680\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-rj49ln\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pc2gx9\\\": PUT https://rp.j4029440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-pc2gx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-rj49ln\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00063aab0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"38dba7f1-fd4e-420c-a529-1198305aac8e\",\n ],\n \"Date\": [\n \"Wed, 25 Mar 2026 00:50:44 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"45\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0006965a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036585018725240832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4603,"pr_state":"open","pr_sha":"fa604f24c12646849b09fba692e1c42b53a8840a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T23:24:40Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f19b54ebd5e3111814758890d21e419e13886ce2298eb563d359d41e651b1ecf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036585018725240832","occurred_at":"2026-03-24T23:24:40Z","failed_at":"e2e","signature_id":"9a4a9a803cf37b77c7f8e5f35f42e619401a914db70c69b39aade7cb7a336849","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c02240\u003e: \n failed to create HCP cluster cluster-zstream-4-19-267c5s: failed starting cluster creation \"cluster-zstream-4-19-267c5s\" in resourcegroup=\"rg-zstream-upgrade-4-19-qmdpmw\": PUT https://rp.j5240832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-qmdpmw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-267c5s\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-267c5s: failed starting cluster creation \\\"cluster-zstream-4-19-267c5s\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-qmdpmw\\\": PUT https://rp.j5240832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-qmdpmw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-267c5s\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c02220\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-267c5s\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-qmdpmw\\\": PUT https://rp.j5240832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-qmdpmw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-267c5s\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010762a0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"46\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"5f4aaf79-a99d-4a76-90aa-286a3071d601\",\n ],\n \"Date\": [\n \"Wed, 25 Mar 2026 00:29:59 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000c02160\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c02240\u003e: \n failed to create HCP cluster cluster-zstream-4-19-267c5s: failed starting cluster creation \"cluster-zstream-4-19-267c5s\" in resourcegroup=\"rg-zstream-upgrade-4-19-qmdpmw\": PUT https://rp.j5240832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-qmdpmw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-267c5s\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-267c5s: failed starting cluster creation \\\"cluster-zstream-4-19-267c5s\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-qmdpmw\\\": PUT https://rp.j5240832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-qmdpmw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-267c5s\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c02220\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-267c5s\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-qmdpmw\\\": PUT https://rp.j5240832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-qmdpmw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-267c5s\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010762a0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"46\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"5f4aaf79-a99d-4a76-90aa-286a3071d601\",\n ],\n \"Date\": [\n \"Wed, 25 Mar 2026 00:29:59 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000c02160\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036578977547554816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T23:00:24Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b008baaa0e6a3f0868756c3ccd52c8cd66eca978e2c7aa4cf9286326bb397866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036578977547554816","occurred_at":"2026-03-24T23:00:24Z","failed_at":"e2e","signature_id":"cfe2ec712227b25399f955211d56c514a91394e5bd00be85923bdb75dd2e0373","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00093d4a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-zbgbs7: failed starting cluster creation \"cluster-zstream-4-19-zbgbs7\" in resourcegroup=\"rg-zstream-upgrade-4-19-jfx54m\": PUT https://rp.j7554816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jfx54m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zbgbs7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-zbgbs7: failed starting cluster creation \\\"cluster-zstream-4-19-zbgbs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jfx54m\\\": PUT https://rp.j7554816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jfx54m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zbgbs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00093d460\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-zbgbs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jfx54m\\\": PUT https://rp.j7554816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jfx54m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zbgbs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00145aae0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"376dae8c-016c-44fa-8968-1e73e43961ce\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:49:01 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"88\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00093d380\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00093d4a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-zbgbs7: failed starting cluster creation \"cluster-zstream-4-19-zbgbs7\" in resourcegroup=\"rg-zstream-upgrade-4-19-jfx54m\": PUT https://rp.j7554816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jfx54m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zbgbs7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-zbgbs7: failed starting cluster creation \\\"cluster-zstream-4-19-zbgbs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jfx54m\\\": PUT https://rp.j7554816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jfx54m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zbgbs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00093d460\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-zbgbs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jfx54m\\\": PUT https://rp.j7554816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jfx54m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zbgbs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00145aae0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"376dae8c-016c-44fa-8968-1e73e43961ce\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:49:01 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"88\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00093d380\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036578831464140800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4614,"pr_state":"closed","pr_sha":"8df604b938b1cb7ca9fc7b71ca465977f708255c","final_merged_sha":"0ea6d9b25300bffbb91053c96049833941c1ccca","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T23:00:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3b67734d5a60828d8a56dc704d90f76e3e18da4c133732746d0bff9764850007","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036578831464140800","occurred_at":"2026-03-24T23:00:02Z","failed_at":"provision","signature_id":"0f8aef3a36dbfe5cfb7bdb1dbd1a5daa057ecfb649fa851f4ef853713a401ad8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/auditLogs/output","test_suite":"step graph","failure_text":"time=2026-03-24T23:35:04.558Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=auditLogs step=output description=\"Step output\\n Kind: ARM\\n Template: templates/output-audit-logs-eventhub.bicep\\n Parameters: configurations/output-audit-logs-eventhub.tmpl.bicepparam\"\ntime=2026-03-24T23:35:04.967Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=auditLogs step=output\ntime=2026-03-24T23:35:05.542Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=auditLogs step=output deployment=1468310f1bc0468f9ed7b0073890fee99bf8c2fff75858f2cf229991d137607d\ntime=2026-03-24T23:35:05.624Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=auditLogs step=output err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/westus3-shared-resources/providers/Microsoft.Resources/deployments/1468310f1bc0468f9ed7b0073890fee99bf8c2fff75858f2cf229991d137607d/operationStatuses/08584272115801546536\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment '1468310f1bc0468f9ed7b0073890fee99bf8c2fff75858f2cf229991d137607d' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036571296334090240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4588,"pr_state":"closed","pr_sha":"afaa40f3c6603ee60e541c85c3e584df93047604","final_merged_sha":"582667ceb51a1eed2a4604a71a440bcf5821ebe2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T22:30:09Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d64d37d879eef6c772e057026a13eda563912361cebd0724e4fe512bc52bdc1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036571296334090240","occurred_at":"2026-03-24T22:30:09Z","failed_at":"e2e","signature_id":"6d4ccf19b3088e26edd3df34ed414ac3f33c9e10d0dd7c1cad83601ef732ea5e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008c5780\u003e: \n failed to create HCP cluster cluster-zstream-4-19-246j5l: failed starting cluster creation \"cluster-zstream-4-19-246j5l\" in resourcegroup=\"rg-zstream-upgrade-4-19-dk77vq\": PUT https://rp.j4090240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-dk77vq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-246j5l\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-246j5l: failed starting cluster creation \\\"cluster-zstream-4-19-246j5l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-dk77vq\\\": PUT https://rp.j4090240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-dk77vq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-246j5l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008c5760\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-246j5l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-dk77vq\\\": PUT https://rp.j4090240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-dk77vq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-246j5l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00131cbd0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"7b77ae63-ce18-45be-b106-3c6176e301eb\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:38:23 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"50\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0008c5660\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008c5780\u003e: \n failed to create HCP cluster cluster-zstream-4-19-246j5l: failed starting cluster creation \"cluster-zstream-4-19-246j5l\" in resourcegroup=\"rg-zstream-upgrade-4-19-dk77vq\": PUT https://rp.j4090240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-dk77vq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-246j5l\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-246j5l: failed starting cluster creation \\\"cluster-zstream-4-19-246j5l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-dk77vq\\\": PUT https://rp.j4090240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-dk77vq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-246j5l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008c5760\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-246j5l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-dk77vq\\\": PUT https://rp.j4090240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-dk77vq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-246j5l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00131cbd0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"7b77ae63-ce18-45be-b106-3c6176e301eb\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:38:23 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"50\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0008c5660\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036571047913852928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4612,"pr_state":"closed","pr_sha":"8914600ae23dd35594345b495720b962fe2b0acf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T22:29:10Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3af00676c5d6a25c88f3bdebf4dbdb7faed341162fc81f1b6a25fb68d0c0c674","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036571047913852928","occurred_at":"2026-03-24T22:29:10Z","failed_at":"provision","signature_id":"5a16efce6856cb7ec3c289414cb5c6df861b61434d949293d8a83736a8fb478f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-24T23:09:24.452Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Shell\\n Command: make deploy\\n\"\ntime=2026-03-24T23:19:44.737Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Shell Step, failed to execute shell command: make[2]: Entering directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\nkubectl create namespace clusters-service --dry-run=client -o json | kubectl apply -f - \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_CLIENT_ID=bba9ecaf-0a5e-47af-8d23-c33286ffeb31 \u0026\u0026 \\\\\\nIMAGE_PULLER_MI_TENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nkubectl label namespace clusters-service \\\"istio.io/rev=prod-stable\\\" --overwrite=true \u0026\u0026 \\\\\\nAZURE_CS_MI_CLIENT_ID=f3698ec3-5fdb-4fee-8a48-711baac420de \u0026\u0026 \\\\\\nTENANT_ID=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \u0026\u0026 \\\\\\nOIDC_BLOB_SERVICE_ENDPOINT=https://arohcpprowoidcj3852928.blob.core.windows.net/ \u0026\u0026 \\\\\\nOIDC_ISSUER_BASE_ENDPOINT=https://arohcpprowoidcj3852928.z1.web.core.windows.net/ \u0026\u0026 \\\\\\nDB_HOST=$(if [ \\\"false\\\" = \\\"true\\\" ]; then az postgres flexible-server show -g hcp-underlay-prow-j3852928 -n arohcp-prow-dbcs-j3852928 --query fullyQualifiedDomainName -o tsv; else echo \\\"ocm-cs-db\\\"; fi) \u0026\u0026 \\\\\\nOVERRIDES=$(if [ \\\"false\\\" = \\\"true\\\" ]; then echo \\\"azuredb.values.yaml\\\"; else echo \\\"containerdb.values.yaml\\\"; fi) \u0026\u0026 \\\\\\nZONE_RESOURCE_ID=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3852928/providers/Microsoft.Network/dnszones/j3852928.hcp.osadev.cloud \u0026\u0026 \\\\\\nCX_SECRETS_KV_URL=\\\"https://ah-prow-cx-j3852928-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\nCX_MI_KV_URL=\\\"https://ah-prow-mi-j3852928-1.vault.azure.net/\\\" \u0026\u0026 \\\\\\n./hack/helm.sh cluster-service helm-charts/cluster-service clusters-service \\\\\\n -f helm-charts/cluster-service/${OVERRIDES} \\\\\\n --set serviceAccountName=clusters-service \\\\\\n --set environment=arohcpprow \\\\\\n --set azureCsMiClientId=${AZURE_CS_MI_CLIENT_ID} \\\\\\n --set oidcIssuerBlobServiceUrl=${OIDC_BLOB_SERVICE_ENDPOINT} \\\\\\n --set oidcIssuerBaseUrl=${OIDC_ISSUER_BASE_ENDPOINT} \\\\\\n --set tlsCertificatesIssuer=${TLS_CERTIFICATES_ISSUER} \\\\\\n --set tlsCertificatesEnabled=true \\\\\\n --set denyAssignments=disabled \\\\\\n --set tenantId=${TENANT_ID} \\\\\\n --set region=westus3 \\\\\\n --set serviceKeyvaultName=aro-hcp-dev-svc-kv \\\\\\n --set replicas=3 \\\\\\n --set imageRegistry=arohcpsvcdev.azurecr.io \\\\\\n --set imageRepository=app-sre/aro-hcp-clusters-service \\\\\\n --set imageDigest=sha256:5f079a0240687e6d2214d892a61019db3a3bbb4e3afb97bcf7731633af4ce7d8 \\\\\\n --set global.imageDigest=sha256:5f079a0240687e6d2214d892a61019db3a3bbb4e3afb97bcf7731633af4ce7d8 \\\\\\n --set azureFirstPartyApplicationClientId=b3cb2fab-15cb-4583-ad06-f91da9bfe2d1 \\\\\\n --set batchProcessesDryRun=true \\\\\\n --set batchProcesses=\\\"\\\" \\\\\\n --set fpaCertName=firstPartyCert2 \\\\\\n --set ocpAcrResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev \\\\\\n --set ocpAcrUrl=arohcpocpdev.azurecr.io \\\\\\n --set shard.consumerName=\\\"hcp-underlay-j3852928-mgmt-1\\\" \\\\\\n --set shard.zoneResourceId=\\\"${ZONE_RESOURCE_ID}\\\" \\\\\\n --set shard.cxSecretsKeyVaultUrl=\\\"${CX_SECRETS_KV_URL}\\\" \\\\\\n --set shard.cxMiKeyVaultUrl=\\\"${CX_MI_KV_URL}\\\" \\\\\\n --set shard.cxSecretsKeyVaultMiClientId=\\\"9a47fa51-2376-4a6c-b822-670de4dd981b\\\" \\\\\\n --set shard.maestroRestUrl=\\\"http://maestro.maestro.svc.cluster.local:8000\\\" \\\\\\n --set shard.maestroGrpUrl=\\\"maestro-grpc.maestro.svc.cluster.local:8090\\\" \\\\\\n --set shard.Id=$(python3 -c \\\"import uuid; print(uuid.uuid5(uuid.UUID('916f9976-e1c0-4afd-b84c-5d5c94fbeaed'), 'prow-j3852928-mgmt-1'))\\\") \\\\\\n --set shard.aksManagementClusterResourceId=/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j3852928-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j3852928-mgmt-1 \\\\\\n --set databaseHost=${DB_HOST} \\\\\\n --set azureMiMockServicePrincipalPrincipalId=d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 \\\\\\n --set azureMiMockServicePrincipalClientId=e8723db7-9b9e-46a4-9f7d-64d75c3534f0 \\\\\\n --set azureMiMockServicePrincipalCertName=msiMockCert2 \\\\\\n --set azureArmHelperIdentityCertName=armHelperCert2 \\\\\\n --set azureArmHelperIdentityClientId=3331e670-0804-48e8-a086-6241671ddc93 \\\\\\n --set azureArmHelperMockFpaPrincipalId=47f69502-0065-4d9a-b19b-d403e183d2f4 \\\\\\n --set azureCloudEnvironmentName=AzurePublicCloud \\\\\\n --set pullBinding.workloadIdentityClientId=\\\"${IMAGE_PULLER_MI_CLIENT_ID}\\\" \\\\\\n --set pullBinding.workloadIdentityTenantId=\\\"${IMAGE_PULLER_MI_TENANT_ID}\\\" \\\\\\n --set pullBinding.registry=arohcpsvcdev.azurecr.io \\\\\\n --set pullBinding.scope=repository:app-sre/aro-hcp-clusters-service:pull \\\\\\n --set managedIdentitiesDataPlaneAudienceResource=https://dummy.org \\\\\\n --set tracing.address=http://ingest.observability:4318 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxSurge=1 \\\\\\n --set csDeploymentStrategy.rollingUpdate.maxUnavailable=0 \\\\\\n --set provisionShardClusterLimit=100 \\\\\\n --set ocpVersions.defaultVersion.version=4.20.8 \\\\\\n --set ocpVersions.channelGroups.stable.minVersion=4.19.0 \\\\\\n --set ocpVersions.channelGroups.stable.maxVersion= \\\\\\n --set ocpVersions.channelGroups.candidate.minVersion=4.19.7 \\\\\\n --set ocpVersions.channelGroups.candidate.maxVersion= \\\\\\n --set ocpVersions.channelGroups.nightly.minVersion=4.19.0-0.nightly-20200101 \\\\\\n --set ocpVersions.channelGroups.nightly.maxVersion= \\\\\\n --set azureOperatorsMI.roleSetName=dev \\\\\\n --set deployment.zoneCount=4\\nnamespace/clusters-service created\\nnamespace/clusters-service labeled\\nRun Helm upgrade with release name cluster-service in namespace clusters-service\\nRelease \\\"cluster-service\\\" does not exist. Installing it now.\\nError: context deadline exceeded\\nmake[2]: *** [Makefile:32: deploy] Error 1\\nmake[2]: Leaving directory '/opt/app-root/src/github.com/Azure/ARO-HCP/cluster-service'\\n exit status 2\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40","failure_pattern":"make[2]: *** [Makefile:32: deploy] Error 1","search_query":"make[2]: *** [Makefile:32: deploy] Error 1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ec25dc4fdc80e4a5974a018bbe111f0574bf241fc82eb76a181a67e25965e40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036570601946091520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"9e233cc5bb9a5a5bdf9a9e643724f961ad1d289d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T22:27:23Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8a3863bf6f333641b724442ab903224fb2945394287e734cddfce53040c0bce3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036570601946091520","occurred_at":"2026-03-24T22:27:23Z","failed_at":"e2e","signature_id":"c5d2907d493f049dbcb5d149f28c3560db869d021d9b7916405d4cdf9d9f9e3e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000493660\u003e: \n failed to create HCP cluster cluster-zstream-4-19-nzpzxv: failed starting cluster creation \"cluster-zstream-4-19-nzpzxv\" in resourcegroup=\"rg-zstream-upgrade-4-19-z99gwj\": PUT https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-z99gwj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-nzpzxv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-nzpzxv: failed starting cluster creation \\\"cluster-zstream-4-19-nzpzxv\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-z99gwj\\\": PUT https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-z99gwj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-nzpzxv\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000493640\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-nzpzxv\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-z99gwj\\\": PUT https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-z99gwj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-nzpzxv\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00140a570\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"7955bfc9-e142-4be0-830d-7b198283f84d\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:32:17 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000493460\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000493660\u003e: \n failed to create HCP cluster cluster-zstream-4-19-nzpzxv: failed starting cluster creation \"cluster-zstream-4-19-nzpzxv\" in resourcegroup=\"rg-zstream-upgrade-4-19-z99gwj\": PUT https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-z99gwj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-nzpzxv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-nzpzxv: failed starting cluster creation \\\"cluster-zstream-4-19-nzpzxv\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-z99gwj\\\": PUT https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-z99gwj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-nzpzxv\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000493640\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-nzpzxv\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-z99gwj\\\": PUT https://rp.j6091520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-z99gwj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-nzpzxv\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00140a570\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"7955bfc9-e142-4be0-830d-7b198283f84d\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:32:17 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000493460\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036567747059519488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"70aec382b41fff05f5e5a9361811a8bc13f282a4","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T22:16:03Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3c551d3298f6f4791b97d0585c96b4761304f717b386be3357ed82165e55760b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036567747059519488","occurred_at":"2026-03-24T22:16:03Z","failed_at":"e2e","signature_id":"186fe4e793dd1f50ff47a3f842c0cc190f46ed871a32870576cb641cbf3c2a9c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e145a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-dclbd6: failed starting cluster creation \"cluster-zstream-4-19-dclbd6\" in resourcegroup=\"rg-zstream-upgrade-4-19-jcrbds\": PUT https://rp.j9519488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jcrbds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-dclbd6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-dclbd6: failed starting cluster creation \\\"cluster-zstream-4-19-dclbd6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jcrbds\\\": PUT https://rp.j9519488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jcrbds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-dclbd6\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e14580\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-dclbd6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jcrbds\\\": PUT https://rp.j9519488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jcrbds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-dclbd6\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012a0c30\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"c75630de-133b-4d88-b74b-a3488c538c90\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:21:11 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"79\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e144c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e145a0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-dclbd6: failed starting cluster creation \"cluster-zstream-4-19-dclbd6\" in resourcegroup=\"rg-zstream-upgrade-4-19-jcrbds\": PUT https://rp.j9519488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jcrbds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-dclbd6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-dclbd6: failed starting cluster creation \\\"cluster-zstream-4-19-dclbd6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jcrbds\\\": PUT https://rp.j9519488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jcrbds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-dclbd6\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e14580\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-dclbd6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-jcrbds\\\": PUT https://rp.j9519488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-jcrbds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-dclbd6\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012a0c30\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"c75630de-133b-4d88-b74b-a3488c538c90\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 23:21:11 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"79\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e144c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036556943501299712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4541,"pr_state":"closed","pr_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","final_merged_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T21:33:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2249987f86e5467ae6092334b40aa9fb21a35f6883d9996c7dc3d62eb54e3fd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036556943501299712","occurred_at":"2026-03-24T21:33:07Z","failed_at":"e2e","signature_id":"776cda67cc909351f3059b0abf97ff52796d80255a7a9859d049ec63b018cb09","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a1bbe0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-pghf64: failed starting cluster creation \"cluster-zstream-4-19-pghf64\" in resourcegroup=\"rg-zstream-upgrade-4-19-cdxzwd\": PUT https://rp.j1299712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cdxzwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pghf64\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-pghf64: failed starting cluster creation \\\"cluster-zstream-4-19-pghf64\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cdxzwd\\\": PUT https://rp.j1299712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cdxzwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pghf64\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a1bbc0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-pghf64\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cdxzwd\\\": PUT https://rp.j1299712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cdxzwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pghf64\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00139dbf0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"c431ec20-b434-44c3-9361-0ee984e17c83\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:43:35 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"45\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a1bac0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a1bbe0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-pghf64: failed starting cluster creation \"cluster-zstream-4-19-pghf64\" in resourcegroup=\"rg-zstream-upgrade-4-19-cdxzwd\": PUT https://rp.j1299712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cdxzwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pghf64\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-pghf64: failed starting cluster creation \\\"cluster-zstream-4-19-pghf64\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cdxzwd\\\": PUT https://rp.j1299712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cdxzwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pghf64\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a1bbc0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-pghf64\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-cdxzwd\\\": PUT https://rp.j1299712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-cdxzwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pghf64\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00139dbf0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"c431ec20-b434-44c3-9361-0ee984e17c83\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:43:35 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"45\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a1bac0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036554691541733376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4515,"pr_state":"closed","pr_sha":"fd99db312afd8c50e15b0c5fafbb2271e2bfe325","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T21:24:10Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1733b862257447b2352244f7125e0a673fd97bcff99d8720b3f3f61e73f4ff0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036554691541733376","occurred_at":"2026-03-24T21:24:10Z","failed_at":"e2e","signature_id":"0261e35922aa445f6f73bf678563c5bdb67ed93442b2690ace9bc5c07454d2b1","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c8e9c0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-fksgqz: failed starting cluster creation \"cluster-zstream-4-19-fksgqz\" in resourcegroup=\"rg-zstream-upgrade-4-19-gxztpf\": PUT https://rp.j1733376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-gxztpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-fksgqz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-fksgqz: failed starting cluster creation \\\"cluster-zstream-4-19-fksgqz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gxztpf\\\": PUT https://rp.j1733376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-gxztpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-fksgqz\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c8e9a0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-fksgqz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gxztpf\\\": PUT https://rp.j1733376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-gxztpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-fksgqz\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001274660\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"18a1547c-f43f-4c50-a88c-24b66dbe1e3a\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:36:41 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1046\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000c8e8a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c8e9c0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-fksgqz: failed starting cluster creation \"cluster-zstream-4-19-fksgqz\" in resourcegroup=\"rg-zstream-upgrade-4-19-gxztpf\": PUT https://rp.j1733376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-gxztpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-fksgqz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-fksgqz: failed starting cluster creation \\\"cluster-zstream-4-19-fksgqz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gxztpf\\\": PUT https://rp.j1733376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-gxztpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-fksgqz\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000c8e9a0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-fksgqz\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-gxztpf\\\": PUT https://rp.j1733376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-gxztpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-fksgqz\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001274660\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"18a1547c-f43f-4c50-a88c-24b66dbe1e3a\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:36:41 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"1046\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000c8e8a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8050ee58abfabaeec0de95d637d34998c13ff42162a56680a843a792d07748ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036554691541733376","occurred_at":"2026-03-24T21:24:10Z","failed_at":"e2e","signature_id":"15675c619bc77e2061eb18aec2a5459cc8eb55c711341e216c0375232a686571","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:276]: Timed out after 300.000s.\nExpected\n \u003cv1.PodPhase\u003e: Running\nto equal\n \u003cv1.PodPhase\u003e: Succeeded\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:276]: Timed out after 300.000s.\nExpected\n \u003cv1.PodPhase\u003e: Running\nto equal\n \u003cv1.PodPhase\u003e: Succeeded","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b2294f64176300144ce8b3080430bffa51c6b57bf80df075d489f3117ec76ca","failure_pattern":"Customer should be able to use workload identity via the cluster OIDC issuer URL: Timed out after \u003cduration\u003es.","search_query":"Timed out after 300.000s."}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7b2294f64176300144ce8b3080430bffa51c6b57bf80df075d489f3117ec76ca","890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036554458514591744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4461,"pr_state":"closed","pr_sha":"9baa329552156f993c1fc5b382b78423cd754fef","final_merged_sha":"9baa329552156f993c1fc5b382b78423cd754fef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T21:23:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c3ef7ae972dfe9265ca89c54fa942236854128091753762207d6a7c0dcda071c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036554458514591744","occurred_at":"2026-03-24T21:23:14Z","failed_at":"e2e","signature_id":"d2011d17e5a84f2f0df56913467a1616bbd38ba09e123e440f22355a3bc1fa1b","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007e2680\u003e: \n failed to create HCP cluster cluster-zstream-4-19-7g9hqt: failed starting cluster creation \"cluster-zstream-4-19-7g9hqt\" in resourcegroup=\"rg-zstream-upgrade-4-19-ld7rf2\": PUT https://rp.j4591744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-ld7rf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-7g9hqt\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-7g9hqt: failed starting cluster creation \\\"cluster-zstream-4-19-7g9hqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-ld7rf2\\\": PUT https://rp.j4591744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-ld7rf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-7g9hqt\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007e2660\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-7g9hqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-ld7rf2\\\": PUT https://rp.j4591744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-ld7rf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-7g9hqt\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0013af0b0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"39\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"1afb2267-84f2-4151-be67-4ec123738dbb\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:36:51 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007e25a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007e2680\u003e: \n failed to create HCP cluster cluster-zstream-4-19-7g9hqt: failed starting cluster creation \"cluster-zstream-4-19-7g9hqt\" in resourcegroup=\"rg-zstream-upgrade-4-19-ld7rf2\": PUT https://rp.j4591744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-ld7rf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-7g9hqt\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-7g9hqt: failed starting cluster creation \\\"cluster-zstream-4-19-7g9hqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-ld7rf2\\\": PUT https://rp.j4591744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-ld7rf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-7g9hqt\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007e2660\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-7g9hqt\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-ld7rf2\\\": PUT https://rp.j4591744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-ld7rf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-7g9hqt\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0013af0b0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"39\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"1afb2267-84f2-4151-be67-4ec123738dbb\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:36:51 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007e25a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036553814831534080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"61fdb103bc95a2dfaef677f5e48c76666352f5b4","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T21:20:41Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f38ef6d9749ef9655c74d53385166f04dc984ae2e7e150bb97af8548d1e0be21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036553814831534080","occurred_at":"2026-03-24T21:20:41Z","failed_at":"e2e","signature_id":"0880baa6e6c3be3c1f47f3489eede56d96e020d3f33e53a28991e43d98a39132","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e590e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-l4l2ck: failed starting cluster creation \"cluster-zstream-4-19-l4l2ck\" in resourcegroup=\"rg-zstream-upgrade-4-19-t6x9rx\": PUT https://rp.j1534080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-t6x9rx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-l4l2ck\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-l4l2ck: failed starting cluster creation \\\"cluster-zstream-4-19-l4l2ck\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-t6x9rx\\\": PUT https://rp.j1534080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-t6x9rx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-l4l2ck\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e590c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-l4l2ck\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-t6x9rx\\\": PUT https://rp.j1534080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-t6x9rx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-l4l2ck\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00140a660\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"45\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"2ebdb90d-96fc-42d7-a5a3-1e6d181a2e73\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:37:04 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e58fc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e590e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-l4l2ck: failed starting cluster creation \"cluster-zstream-4-19-l4l2ck\" in resourcegroup=\"rg-zstream-upgrade-4-19-t6x9rx\": PUT https://rp.j1534080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-t6x9rx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-l4l2ck\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-l4l2ck: failed starting cluster creation \\\"cluster-zstream-4-19-l4l2ck\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-t6x9rx\\\": PUT https://rp.j1534080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-t6x9rx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-l4l2ck\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e590c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-l4l2ck\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-t6x9rx\\\": PUT https://rp.j1534080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-t6x9rx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-l4l2ck\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00140a660\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"45\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"2ebdb90d-96fc-42d7-a5a3-1e6d181a2e73\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:37:04 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e58fc0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036547631462748160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T20:56:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a54cbdce4f00cf9273fc9ec4dd1bb08afa486feda1082d8f3c8279b6e6e933b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036547631462748160","occurred_at":"2026-03-24T20:56:00Z","failed_at":"e2e","signature_id":"d4557b62504e603a13339528836f2ede8d01f3fc124783eb2585b04e8f4bd5a3","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a10f00\u003e: \n failed to create HCP cluster cluster-zstream-4-19-jvdvs7: failed starting cluster creation \"cluster-zstream-4-19-jvdvs7\" in resourcegroup=\"rg-zstream-upgrade-4-19-fmtc66\": PUT https://rp.j2748160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-fmtc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jvdvs7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-jvdvs7: failed starting cluster creation \\\"cluster-zstream-4-19-jvdvs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-fmtc66\\\": PUT https://rp.j2748160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-fmtc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jvdvs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a10ee0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-jvdvs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-fmtc66\\\": PUT https://rp.j2748160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-fmtc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jvdvs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00137aab0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"30abdb1c-3027-441a-afe4-0c56dfefc480\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:03:52 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"55\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a10e20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a10f00\u003e: \n failed to create HCP cluster cluster-zstream-4-19-jvdvs7: failed starting cluster creation \"cluster-zstream-4-19-jvdvs7\" in resourcegroup=\"rg-zstream-upgrade-4-19-fmtc66\": PUT https://rp.j2748160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-fmtc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jvdvs7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-jvdvs7: failed starting cluster creation \\\"cluster-zstream-4-19-jvdvs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-fmtc66\\\": PUT https://rp.j2748160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-fmtc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jvdvs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a10ee0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-jvdvs7\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-fmtc66\\\": PUT https://rp.j2748160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-fmtc66/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jvdvs7\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00137aab0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"30abdb1c-3027-441a-afe4-0c56dfefc480\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 22:03:52 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"55\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a10e20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036543399229657088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T20:39:18Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"73d89372383918cc5ebc488c3b094a5a17f109ab636286b604b64a910a063b20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036543399229657088","occurred_at":"2026-03-24T20:39:18Z","failed_at":"e2e","signature_id":"171d21cde723d0245055612c155859eebe6f23489044cd6908ee0a9140e910ef","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000759300\u003e: \n failed to create HCP cluster cluster-zstream-4-19-zvpj7f: failed starting cluster creation \"cluster-zstream-4-19-zvpj7f\" in resourcegroup=\"rg-zstream-upgrade-4-19-nrk9nl\": PUT https://rp.j9657088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nrk9nl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zvpj7f\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-zvpj7f: failed starting cluster creation \\\"cluster-zstream-4-19-zvpj7f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-nrk9nl\\\": PUT https://rp.j9657088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nrk9nl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zvpj7f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007592e0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-zvpj7f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-nrk9nl\\\": PUT https://rp.j9657088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nrk9nl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zvpj7f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000b6ec60\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"43\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"08392dbf-202a-47fa-8b39-2f441a2a76f8\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:44:37 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007591a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000759300\u003e: \n failed to create HCP cluster cluster-zstream-4-19-zvpj7f: failed starting cluster creation \"cluster-zstream-4-19-zvpj7f\" in resourcegroup=\"rg-zstream-upgrade-4-19-nrk9nl\": PUT https://rp.j9657088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nrk9nl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zvpj7f\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-zvpj7f: failed starting cluster creation \\\"cluster-zstream-4-19-zvpj7f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-nrk9nl\\\": PUT https://rp.j9657088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nrk9nl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zvpj7f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007592e0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-zvpj7f\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-nrk9nl\\\": PUT https://rp.j9657088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-nrk9nl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-zvpj7f\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000b6ec60\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"43\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"08392dbf-202a-47fa-8b39-2f441a2a76f8\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:44:37 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007591a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036542725548937216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T20:36:37Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c88ea43ca89870ffd038c340c60e29bdaa8bf0d4ec7173623487c4e38edf5fb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036542725548937216","occurred_at":"2026-03-24T20:36:37Z","failed_at":"e2e","signature_id":"393d943d9ec6d5b2a7df2da319288c3c8735ecff2fea028f21c2789b49797092","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007d97e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-g8q84l: failed starting cluster creation \"cluster-zstream-4-19-g8q84l\" in resourcegroup=\"rg-zstream-upgrade-4-19-2s97kp\": PUT https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-2s97kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g8q84l\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-g8q84l: failed starting cluster creation \\\"cluster-zstream-4-19-g8q84l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-2s97kp\\\": PUT https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-2s97kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g8q84l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007d97c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-g8q84l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-2s97kp\\\": PUT https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-2s97kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g8q84l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012f8fc0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"a749caaf-f103-4471-9de3-54f2e13526a1\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:48:10 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"69\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007d96a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007d97e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-g8q84l: failed starting cluster creation \"cluster-zstream-4-19-g8q84l\" in resourcegroup=\"rg-zstream-upgrade-4-19-2s97kp\": PUT https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-2s97kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g8q84l\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-g8q84l: failed starting cluster creation \\\"cluster-zstream-4-19-g8q84l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-2s97kp\\\": PUT https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-2s97kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g8q84l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007d97c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-g8q84l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-2s97kp\\\": PUT https://rp.j8937216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-2s97kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-g8q84l\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012f8fc0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"a749caaf-f103-4471-9de3-54f2e13526a1\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:48:10 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"69\"],\n \"Server\": [\"istio-envoy\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0007d96a0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036539966367993856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"4def80b288359cfb70b6f3dd10a3a52aaed55751","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T20:25:33Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"24e0c0c6dca0ee4464a436f4720becc36f1bd7800db3ae4ce842d07caadf8442","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036539966367993856","occurred_at":"2026-03-24T20:25:33Z","failed_at":"e2e","signature_id":"d559626bd4e8986d1bca860171d7ede03bc9d2427bb20975f13172219ff86acb","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:143]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00141aea0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vl8jff-6ld5tp, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-vl8jff\" in resourcegroup=\"rg-np-version-upgrade-vl8jff-6ld5tp\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vl8jff-6ld5tp, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vl8jff\\\" in resourcegroup=\\\"rg-np-version-upgrade-vl8jff-6ld5tp\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005c2e60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vl8jff-6ld5tp\",\n },\n \u003c*fmt.wrapError | 0xc000a9b0c0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vl8jff\\\" in resourcegroup=\\\"rg-np-version-upgrade-vl8jff-6ld5tp\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:143]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00141aea0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vl8jff-6ld5tp, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-vl8jff\" in resourcegroup=\"rg-np-version-upgrade-vl8jff-6ld5tp\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vl8jff-6ld5tp, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vl8jff\\\" in resourcegroup=\\\"rg-np-version-upgrade-vl8jff-6ld5tp\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005c2e60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vl8jff-6ld5tp\",\n },\n \u003c*fmt.wrapError | 0xc000a9b0c0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vl8jff\\\" in resourcegroup=\\\"rg-np-version-upgrade-vl8jff-6ld5tp\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-c9df8c\" in resourcegroup=\"rg-np-version-upgrade-c9df8c-4rs6vp\" to finish creating: context deadline exceeded"}},{"row_id":"9505e848567e88715045757b0d0e030a163e0e0227f14f64e5c631585b31b529","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036539966367993856","occurred_at":"2026-03-24T20:25:33Z","failed_at":"e2e","signature_id":"12010b0dce39042cec3ee08fa5b56e05cb2e4c883186f8865f72a52a274ffb8d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000318400\u003e: \n failed to create HCP cluster cluster-zstream-4-19-kbdz2n: failed starting cluster creation \"cluster-zstream-4-19-kbdz2n\" in resourcegroup=\"rg-zstream-upgrade-4-19-j8vzrp\": PUT https://rp.j7993856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-j8vzrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-kbdz2n\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-kbdz2n: failed starting cluster creation \\\"cluster-zstream-4-19-kbdz2n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-j8vzrp\\\": PUT https://rp.j7993856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-j8vzrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-kbdz2n\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0003183e0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-kbdz2n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-j8vzrp\\\": PUT https://rp.j7993856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-j8vzrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-kbdz2n\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000de2b10\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"4b295f70-546d-482a-95da-14e969a9d864\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:38:55 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"61\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000318320\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000318400\u003e: \n failed to create HCP cluster cluster-zstream-4-19-kbdz2n: failed starting cluster creation \"cluster-zstream-4-19-kbdz2n\" in resourcegroup=\"rg-zstream-upgrade-4-19-j8vzrp\": PUT https://rp.j7993856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-j8vzrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-kbdz2n\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-kbdz2n: failed starting cluster creation \\\"cluster-zstream-4-19-kbdz2n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-j8vzrp\\\": PUT https://rp.j7993856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-j8vzrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-kbdz2n\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0003183e0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-kbdz2n\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-j8vzrp\\\": PUT https://rp.j7993856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-j8vzrp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-kbdz2n\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000de2b10\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"4b295f70-546d-482a-95da-14e969a9d864\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:38:55 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n \"X-Envoy-Upstream-Service-Time\": [\"61\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000318320\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a99c39ff4b2034dc180baecc798ae04f953c584e45809cad63eeebc08033da26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036539966367993856","occurred_at":"2026-03-24T20:25:33Z","failed_at":"e2e","signature_id":"f4f94d3564a40b451858cf370079b1b898c5dc22466e328c004eeed80be3cdec","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:117]: Timed out after 2400.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ccd290\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \"4.21.5\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00099e540\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n err: \u003c*errors.errorString | 0xc0009b2510\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:117]: Timed out after 2400.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ccd290\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \"4.21.5\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00099e540\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n err: \u003c*errors.errorString | 0xc0009b2510\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036537613992595456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4603,"pr_state":"open","pr_sha":"fa604f24c12646849b09fba692e1c42b53a8840a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T20:16:18Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d28c62869aeece8398d509270bcf8e0c032b48828261aa0824e858bad5b4c889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036537613992595456","occurred_at":"2026-03-24T20:16:18Z","failed_at":"e2e","signature_id":"7441ddb052c86e730301552122531681cf939122ea0f295350c8acc2660bb7c4","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0002f42e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-jp64d5: failed starting cluster creation \"cluster-zstream-4-19-jp64d5\" in resourcegroup=\"rg-zstream-upgrade-4-19-vmnpdh\": PUT https://rp.j2595456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-vmnpdh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jp64d5\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-jp64d5: failed starting cluster creation \\\"cluster-zstream-4-19-jp64d5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-vmnpdh\\\": PUT https://rp.j2595456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-vmnpdh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jp64d5\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0002f42c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-jp64d5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-vmnpdh\\\": PUT https://rp.j2595456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-vmnpdh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jp64d5\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000bc8540\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"94\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"ae348005-c559-47a1-a344-e4f5a595dd0e\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:18:09 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0002f40c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0002f42e0\u003e: \n failed to create HCP cluster cluster-zstream-4-19-jp64d5: failed starting cluster creation \"cluster-zstream-4-19-jp64d5\" in resourcegroup=\"rg-zstream-upgrade-4-19-vmnpdh\": PUT https://rp.j2595456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-vmnpdh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jp64d5\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.19.3-candidate' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-jp64d5: failed starting cluster creation \\\"cluster-zstream-4-19-jp64d5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-vmnpdh\\\": PUT https://rp.j2595456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-vmnpdh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jp64d5\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0002f42c0\u003e{\n msg: \"failed starting cluster creation \\\"cluster-zstream-4-19-jp64d5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-vmnpdh\\\": PUT https://rp.j2595456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-vmnpdh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-jp64d5\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Version 'openshift-v4.19.3-candidate' doesn't exist\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000bc8540\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"94\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"ae348005-c559-47a1-a344-e4f5a595dd0e\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 21:18:09 GMT\",\n ],\n \"Content-Length\": [\"138\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0002f40c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 86, 101, 114, 115, 105, 111, 110, 32, 39, 111, 112, 101, 110, 115, 104, 105, 102, 116, 45, 118, 52, 46, 49, 57, 46, 51, 45, 99, 97, 110, 100, 105, 100, 97, 116, 101, 39, 32, 100, 111, 101, 115, 110, 39, 116, 32, 101, 120, 105, 115, 116, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 138,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["890171068940b1511ca1957677f5c5b7c52ba1ddc2dc17407652ef2422442537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036526972728774656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4588,"pr_state":"closed","pr_sha":"afaa40f3c6603ee60e541c85c3e584df93047604","final_merged_sha":"582667ceb51a1eed2a4604a71a440bcf5821ebe2","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T19:33:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036523441913335808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4610,"pr_state":"open","pr_sha":"428d5acda06ee272961298e9e99621f21bd67e05","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T19:19:59Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1b13854bcf04b343c11ce6a5fcb9d8e72abdb571b9072190c1085a78b1087e74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036523441913335808","occurred_at":"2026-03-24T19:19:59Z","failed_at":"e2e","signature_id":"940071d44d10f00346ae006061f759940cf57a14a9a63d86b2e36885bb8cf574","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:145]: cluster Properties.API.URL was nil\nExpected\n \u003c*string | 0x0\u003e: nil\nnot to be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:145]: cluster Properties.API.URL was nil\nExpected\n \u003c*string | 0x0\u003e: nil\nnot to be nil","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f8c3e29686732112a6ec0e7812c19d266182eb637ee1bfa8e39418c748743205","failure_pattern":"cluster Properties.API.URL was nil","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:145]: cluster Properties.API.URL was nil"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f8c3e29686732112a6ec0e7812c19d266182eb637ee1bfa8e39418c748743205"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036523318634352640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4566,"pr_state":"open","pr_sha":"85cd7039419e1de084037498a4ef0875e4587eaf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T19:19:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036514132663996416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T18:43:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036507632851750912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"4def80b288359cfb70b6f3dd10a3a52aaed55751","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T18:17:10Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"68ce0b363719db51971d42e72f978bfa4d24f6627bd2ea94bbcd3f0b1367cb5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036507632851750912","occurred_at":"2026-03-24T18:17:10Z","failed_at":"e2e","signature_id":"0f2ad74a08422a4a4f663878ac41fe824730f5f4f0816bda0ed9d0ca7cdaab2c","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:117]: Timed out after 2400.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f3fec0\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \"4.21.5\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0004657a0\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n err: \u003c*errors.errorString | 0xc000a1e7d0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:117]: Timed out after 2400.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f3fec0\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \"4.21.5\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc0004657a0\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.5) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n err: \u003c*errors.errorString | 0xc000a1e7d0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.5\\\"\",\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"e118ef351a1a10cc9ce584b4d87a6aa300d70b07bfb28ed5b58c94675f86b7e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036507632851750912","occurred_at":"2026-03-24T18:17:10Z","failed_at":"e2e","signature_id":"c6f2f0e60ab7c65a45cc5568afd929a3fa838a0c5455dad5149caad7f9c17387","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:143]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015a88d0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-c9df8c\" in resourcegroup=\"rg-np-version-upgrade-c9df8c-4rs6vp\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-c9df8c\\\" in resourcegroup=\\\"rg-np-version-upgrade-c9df8c-4rs6vp\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a896c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp\",\n },\n \u003c*fmt.wrapError | 0xc0001706a0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-c9df8c\\\" in resourcegroup=\\\"rg-np-version-upgrade-c9df8c-4rs6vp\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:143]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015a88d0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-c9df8c\" in resourcegroup=\"rg-np-version-upgrade-c9df8c-4rs6vp\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-c9df8c\\\" in resourcegroup=\\\"rg-np-version-upgrade-c9df8c-4rs6vp\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a896c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp\",\n },\n \u003c*fmt.wrapError | 0xc0001706a0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-c9df8c\\\" in resourcegroup=\\\"rg-np-version-upgrade-c9df8c-4rs6vp\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-c9df8c-4rs6vp, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-c9df8c\" in resourcegroup=\"rg-np-version-upgrade-c9df8c-4rs6vp\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036502540912168960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4599,"pr_state":"closed","pr_sha":"60ec28b9a14e5c0d6c52a4437eb85f66ef1e370b","final_merged_sha":"f6ed2d92c3031e65888f81d9423e89db816cc046","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T17:56:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036495730943725568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T17:29:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036495169305448448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4607,"pr_state":"closed","pr_sha":"5e2c098d6baefc64fcb6a59ff40528f5123bb971","final_merged_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T17:27:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036487994562580480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"4def80b288359cfb70b6f3dd10a3a52aaed55751","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T16:52:15Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"085064cbe06cfe5dd934ab095c5badcb2773ed1e01dd329d8a2551e9e5729a6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036487994562580480","occurred_at":"2026-03-24T16:52:15Z","failed_at":"provision","signature_id":"dda47655c9e2e96cb46486352e79d764e9168994a11f83337cbb5c897327247a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-24T17:34:18.969Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-24T17:34:29.061Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org\\nFetch pull secret for source registry registry.k8s.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.k8s.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 to arohcpsvcdev.azurecr.io/secrets-store-sync/controller:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5.\\nThe image will still be available under it's original digest sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 in the target registry.\\nError: failed to resolve sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5: GET \\\"https://us-central1-docker.pkg.dev/v2/k8s-artifacts-prod/images/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": response status code 429: toomanyrequests: Quota exceeded for quota metric 'Requests per project per region' and limit 'Requests per project per region per minute per region' of service 'artifactregistry.googleapis.com' for consumer 'project_number:388270116193'.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac0b293efebf0dd8798f8b2b3da46915c252677f68f9aebc91ce5710a743a7e9","failure_pattern":"Error: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": response status code 429: toomanyrequests: Quota exceeded for quota metric 'Requests per project per region' and limit 'Requests per project per region per minute per region' of service","search_query":"Error: failed to resolve sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5: GET \\\"https://us-central1-docker.pkg.dev/v2/k8s-artifacts-prod/images/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": response status code 429: toomanyrequests: Quota exceeded for quota metric 'Requests per project per region' and limit 'Requests per project per region per minute per region' of service 'artifactregistry.googleapis.com' for consumer 'project_number:388270116193'."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ac0b293efebf0dd8798f8b2b3da46915c252677f68f9aebc91ce5710a743a7e9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036483784685129728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T16:40:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ddbf7b3299430260c7ce2e7cf937e753bd09fba80290bf048f6c14281b60e889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036483784685129728","occurred_at":"2026-03-24T16:40:14Z","failed_at":"e2e","signature_id":"1d8b5e42edf11aa12ab267e57a3321c088e0b55ed52f532b4715f3e04ad68cf0","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015a8f00\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-wf2tbz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-wf2tbz, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-wf2tbz\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-wf2tbz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00063e230\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-wf2tbz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015a8f00\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-wf2tbz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-wf2tbz, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-wf2tbz\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-wf2tbz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00063e230\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-wf2tbz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036484218350997504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T16:37:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036481930744041472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T16:34:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036481803883122688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"9e233cc5bb9a5a5bdf9a9e643724f961ad1d289d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T16:34:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036485085254914048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"98f74184373832a000db1ebed0d4d6488608e070","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T16:33:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036482172088487937","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4597,"pr_state":"closed","pr_sha":"bd2f0e6c361bf3b68e8bce0c60fb9c8be4f1300a","final_merged_sha":"bd2f0e6c361bf3b68e8bce0c60fb9c8be4f1300a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T16:33:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036481090658504704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4257,"pr_state":"closed","pr_sha":"ff7df37a61db103303b8026aab219da876717393","final_merged_sha":"f9b0dd45c8c2499c52d47ec79f8297cdae4dfe1f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T16:09:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"30e4cfaa3c043b76e8341b640ff35eafacc87aff0ba2321aa928562517bcdbc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4257/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036481090658504704","occurred_at":"2026-03-24T16:09:55Z","failed_at":"provision","signature_id":"72ebf3c186b29e1a8d0c8e9436db66d108adb12e711e2e069aaa5db300b129a3","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Frontend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-24T17:29:21.736Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-24T17:29:22.375Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy values=\"map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j8504704.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:6f21f871-04b5-4824-8b7a-450cf197cf47 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j8504704 databaseUrl:https://arohcpprow-rp-j8504704.documents.azure.com:443/ frontendMiClientId:7b104339-222d-48cb-a095-d5a2d2b28c21 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j8504704] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-zzwvt2zp/pipeline@sha256:f136210a840ccb8ad45d2c57a2f5d3997d2770bcda339b150295813991463c3d replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-zzwvt2zp/pipeline:pull workloadIdentityClientId:e917b0cb-bbfd-4444-ae15-0d5594f67823 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:7b104339-222d-48cb-a095-d5a2d2b28c21 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j8504704.hcpsvc.osadev.cloud]]\"\ntime=2026-03-24T17:29:22.375Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-24T17:29:22.444Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-24T17:29:22.582Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-24T17:29:22.582Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-24T17:29:22.715Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-24T17:34:27.015Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-24T17:34:27.015Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-24T17:34:27.015Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-24T17:34:31.048Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy release=aro-hcp-frontend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-frontend-dev\\\" failed: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j8504704.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:6f21f871-04b5-4824-8b7a-450cf197cf47 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j8504704 databaseUrl:https://arohcpprow-rp-j8504704.documents.azure.com:443/ frontendMiClientId:7b104339-222d-48cb-a095-d5a2d2b28c21 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j8504704] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-zzwvt2zp/pipeline@sha256:f136210a840ccb8ad45d2c57a2f5d3997d2770bcda339b150295813991463c3d replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-zzwvt2zp/pipeline:pull workloadIdentityClientId:e917b0cb-bbfd-4444-ae15-0d5594f67823 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:7b104339-222d-48cb-a095-d5a2d2b28c21 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j8504704.hcpsvc.osadev.cloud]]\"\ntime=2026-03-24T17:34:31.051Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5xUTW8TQQy9768Y7amRdqqyragI6qEfiAMqigTiUMRhdsbZuJ21Vx5vQlF/PNqmIduQhtDrs9/zs8fjLIIagcSdeEjmzASnTl0V4YCrW/D6CSmMkwpSXZgl9Nk18AeCOdAjklrnV/Doe2aMMfkl0xTra9fmhcmnwqRAwfpHtIecsJ35Ni+W6R+dwsLdDyK2XkIWfioIufgYu0sWkyJbpFogpRX/C3gBnQjPMYBcRteHBoWTb//Blzl6uGZCZRnaWElsMf0NRTsXn7hD0jzt4k04XGGSrlVkuuhCDbpnxadS595zRzrs8OXk/5Pelm0bUEGftrCuoI183wDt28FzwnoxQBSn6J2CFZgKpBnIzhc79zLpYrxACkj1M7G2i9FW68CGhfNOZyz4y/Xjn3BEv9y7GHlhXWiQrGvRKu/qY6fI07xezSfW2Su8b8zp770DkF4ASPtJI9O+r539eJ/ddRUIgUL60H/8lD2YxQwEjGIDSV3Tmgp0AUDmIDiFHj7Iy6PyrT06tuXJ1zen4/LduCxv8pE5PDQv5RyfjMvTm3w0yh7MLSOZO6RwhkQgg3PFZNZXanieNu9S9mBa4T66NlqYlsMyeT+RwkSuC9NASq7uJVkCiKnuB80HSP53AAAA//8soq4JVAUAAA==\"\ntime=2026-03-24T17:34:31.055Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy\ntime=2026-03-24T17:34:31.055Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"42454831adde3c75179282c1ce6d94be508cc22e8880a9bd19abc0e54e5a4d48","failure_pattern":"resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["42454831adde3c75179282c1ce6d94be508cc22e8880a9bd19abc0e54e5a4d48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036470012104937472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4477,"pr_state":"closed","pr_sha":"0b221bd4880737db45b2ec3caa97ebf1dd177616","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T15:47:41Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f4fff690819ccb4e15785a6ab272726075472f3d3138cd3c67594e1d2f99c6b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036470012104937472","occurred_at":"2026-03-24T15:47:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4605/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036467978886713344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4605,"pr_state":"closed","pr_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","final_merged_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T15:38:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036468950518206464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4596,"pr_state":"closed","pr_sha":"67bf3e249451c317252c827c70909873b7b8b0e8","final_merged_sha":"67bf3e249451c317252c827c70909873b7b8b0e8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T15:37:36Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3c3a76e1f4497a717639e3fa049b03105d94b664176bd10879bca26e26bdd9dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036468950518206464","occurred_at":"2026-03-24T15:37:36Z","failed_at":"e2e","signature_id":"798d51d245f33194e02dbffb2bde0d8bcded48adb87e6e3af7ec63f7fb8b7d7c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00076a140\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc0009f6a50\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000e343c0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc00081a500\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00076a140\u003e: \n route host was never found: Get \"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud on 172.30.0.10:53: no such host\n {\n msg: \"route host was never found: Get \\\"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\\\": dial tcp: lookup agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud on 172.30.0.10:53: no such host\",\n err: \u003c*url.Error | 0xc0009f6a50\u003e{\n Op: \"Get\",\n URL: \"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\",\n Err: \u003c*net.OpError | 0xc000e343c0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xc00081a500\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f","failure_pattern":"route host was never found: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"route host was never found: Get \"https://agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud\": dial tcp: lookup agnhost-e2e-serving-app-p5wcz.apps.aro.w2u4i3e8k2c2x4k.aukg.j8206464.hcp.osadev.cloud on 172.30.0.10:53: no such host"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd93f6786bdbe9aeb08610dcc4ec01a349d1a8e658826e9448f8193966d5a67f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036466088526483456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T15:31:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4393/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036467614510747648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4393,"pr_state":"closed","pr_sha":"7cddbd3cbc43f559f3df12084ec577bed405c99f","final_merged_sha":"4fe7725f48a2f3f1c0ac843b4434f502807b38e5","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T15:31:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036465121550667776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4513,"pr_state":"open","pr_sha":"faaaa2b05bfdecb774f1ca8d97097f46492cc250","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T15:26:44Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2a35e5b45500eedbf5046aa5e9bab87119af2c6ddabc3a30d541fa9ed75b0044","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4513/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036465121550667776","occurred_at":"2026-03-24T15:26:44Z","failed_at":"e2e","signature_id":"d7cd9905cb0a399cf4696ff51a62e7f8dae533b575932fb6e74fbc26b4ad9bbb","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00087b2f0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-vs482z\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vs482z, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-vs482z\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vs482z, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f77b50\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vs482z\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00087b2f0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-vs482z\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vs482z, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-vs482z\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vs482z, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f77b50\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-vs482z\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036461821505835008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4582,"pr_state":"closed","pr_sha":"d08cdff5478eedf3a5fab8f8c7352207f2c1ef23","final_merged_sha":"453429fc550d57d7915964340a777a91b19a1208","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T15:14:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036462511921827840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4597,"pr_state":"closed","pr_sha":"a053b3997a147623290da32fbd0c5105a0ff248d","final_merged_sha":"bd2f0e6c361bf3b68e8bce0c60fb9c8be4f1300a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T15:09:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036461406223601664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4604,"pr_state":"closed","pr_sha":"fffb2dd1e7da62733bd05b2ec5b1da54f47e6a8b","final_merged_sha":"a1dbbe7bf80566749053e62f1021d697e247684a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T15:00:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036455853883658240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T14:51:11Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c5a1045926d3821faf15c5a88261674cdabc00c488b6f50c9c60e00898d41dbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036455853883658240","occurred_at":"2026-03-24T14:51:11Z","failed_at":"e2e","signature_id":"cf428bc769f08e04d79fe2c4090c60b6eb79169701d93cb0aab1e58183161a69","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000513230\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qk9ftl\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qk9ftl, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-qk9ftl\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qk9ftl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002f9b00\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qk9ftl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000513230\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qk9ftl\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qk9ftl, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-qk9ftl\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qk9ftl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002f9b00\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qk9ftl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036455114696298496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T14:48:19Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"837ebe1127805128791e674c7a51802dc06564484b77f4b8cdd04e759142f7d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036455114696298496","occurred_at":"2026-03-24T14:48:19Z","failed_at":"e2e","signature_id":"260afbd92105d0d804abbebb6ab0052182b0b20d9e9a98d69a60867ef10421fe","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0005ffbf0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bwmxx9\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bwmxx9, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-bwmxx9\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bwmxx9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a05670\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bwmxx9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0005ffbf0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bwmxx9\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bwmxx9, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-bwmxx9\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bwmxx9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a05670\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bwmxx9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036455228424851456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T14:45:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e994a9323a0b4d6dba7843c886f87c76d770423f6a22e7e0fe43bbd1effa72c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036455228424851456","occurred_at":"2026-03-24T14:45:25Z","failed_at":"e2e","signature_id":"ef7cfc17b429732009e4f2c41307b4207b1ce292db6fcd532ca127483b478c92","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000972d50\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mxd8xb\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mxd8xb, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-mxd8xb\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mxd8xb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001643310\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mxd8xb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000972d50\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mxd8xb\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mxd8xb, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-mxd8xb\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mxd8xb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001643310\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mxd8xb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036450840276897792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4602,"pr_state":"closed","pr_sha":"4a35705695b42f3d631e22d84e6cec299431574a","final_merged_sha":"4a35705695b42f3d631e22d84e6cec299431574a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T14:31:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036451547763707904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4603,"pr_state":"open","pr_sha":"969519971aa183d6a0f370811deb1a01f27589c7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T14:23:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036442230792654848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"6b3bb4096dc8a5d39165e12cd12071a33ce50a7e","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T13:56:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036441760195940352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4554,"pr_state":"closed","pr_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","final_merged_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T13:55:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036441395656396800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T13:53:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036435928729260032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4601,"pr_state":"closed","pr_sha":"eec30d3130530d4de45de55d7a0842d3ecaf04cc","final_merged_sha":"d4656bcc056e17f94a38d7d26dc82809bf007acc","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T13:32:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"31aaac9d8ce5114994ebb04a973a1f7f97e792c613c8acee5e6d1ecebd84e6f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036435928729260032","occurred_at":"2026-03-24T13:32:13Z","failed_at":"provision","signature_id":"acf2a29fde50ea33ee023eedb22dad5e84deb2dafecae98a30255dd93333f489","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","test_suite":"step graph","failure_text":"time=2026-03-24T14:08:44.893Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter description=\"Step aro-hcp-exporter\\n Kind: Helm\\n\"\ntime=2026-03-24T14:08:45.577Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter values=\"map[cacheTTL:60m clusterNames:prow-j9260032-svc,prow-j9260032-mgmt-1 collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:service-tag-usage,kusto-logs-current image:map[digest:sha256:38294123323d6fd6b64bf8095ad5cb1d11997023e20d61349d47f6615318205e registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] kustRegion:East US 2 kustoCluster:hcp-dev-us-2 listenAddress::8080 service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:c43973ee-69d4-4271-a612-6159f430ce81 name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-24T14:08:45.577Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-24T14:08:45.762Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter namespace=aro-hcp-exporter\ntime=2026-03-24T14:08:46.066Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter dryRun=false\ntime=2026-03-24T14:08:46.066Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-24T14:08:46.187Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-24T14:13:49.471Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-24T14:13:49.471Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-24T14:13:49.471Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-24T14:13:51.743Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter release=aro-hcp-exporter namespace=aro-hcp-exporter status=failed description=\"Release \\\"aro-hcp-exporter\\\" failed: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[cacheTTL:60m clusterNames:prow-j9260032-svc,prow-j9260032-mgmt-1 collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:service-tag-usage,kusto-logs-current image:map[digest:sha256:38294123323d6fd6b64bf8095ad5cb1d11997023e20d61349d47f6615318205e registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] kustRegion:East US 2 kustoCluster:hcp-dev-us-2 listenAddress::8080 service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:c43973ee-69d4-4271-a612-6159f430ce81 name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-24T14:13:51.744Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yRT0sDMRDF7/sphj11Ybf0z1J0pQdBT6IXPVU8ZJNHm3Y3E5Jpa6EfXtairULBev1l3ksyv6SBUEDkddCINCWjRImqG/S4XkLLg3WmihKsm+d0QE+qxTfCBu6TRK/0F85eEyKi9BlhYzUe2VnhkOaUqsDFQvsC756D4Az7kb7VmtdO/pm+NHYH3/CuxQUXJm83yWpdIzgI4n23kJjsabtAAIltEUW1nmrIFnDUM0rQ4V46GowmxWBcjMqXYVkNrqpyMksz6vfp3MxwXJXXszTLkj0t2TpaWWem1jmEE43s6GjvVNtvX8mefODu9PjQnDybw/DfSnJqeJ5TixjVvKvkYBCo3p183iDqjwAAAP//XMOyfGwCAAA=\"\ntime=2026-03-24T14:13:51.746Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-03-24T14:13:51.746Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25","failure_pattern":"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036432645897850880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4597,"pr_state":"closed","pr_sha":"74cb42ac5c54e68d8be0987ddb827526ffc03a30","final_merged_sha":"bd2f0e6c361bf3b68e8bce0c60fb9c8be4f1300a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T13:19:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036425797165649920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"96e81dadff454ff306a80460fd99b013a43d4329","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T12:51:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036422172645265408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4600,"pr_state":"closed","pr_sha":"7de281f726d6f89eece80a8272da813d67560817","final_merged_sha":"7de281f726d6f89eece80a8272da813d67560817","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T12:37:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036420679103942656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4576,"pr_state":"closed","pr_sha":"4eda0032750666275592116643efcf64bdf1b330","final_merged_sha":"4eda0032750666275592116643efcf64bdf1b330","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T12:31:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036419864997924864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4592,"pr_state":"closed","pr_sha":"4bcc7ca5dce31f4cecce91af4a237c05cecd7237","final_merged_sha":"4bcc7ca5dce31f4cecce91af4a237c05cecd7237","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T12:28:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036413897480605696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T12:04:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036411664005337088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T11:55:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036410979692056576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T11:53:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036408642898497536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4599,"pr_state":"closed","pr_sha":"7bdf0d998b205482ef8b377821bc2f22f5a1d50c","final_merged_sha":"f6ed2d92c3031e65888f81d9423e89db816cc046","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T11:43:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036407117652430848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4598,"pr_state":"closed","pr_sha":"724c3cb17a7102d1e0e04506659d9a60d48f6932","final_merged_sha":"724c3cb17a7102d1e0e04506659d9a60d48f6932","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T11:37:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036407032386424832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4554,"pr_state":"closed","pr_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","final_merged_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T11:37:22Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dbfaf1b520a0689e63f4927749114c4202761a3009f505d40e5850bb1f77f4d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036407032386424832","occurred_at":"2026-03-24T11:37:22Z","failed_at":"e2e","signature_id":"48ca49f80e3f8eb2dd47f1a11ee1780349e27a0ef3171bde9e03b37b6ff29e67","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f832c0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-lpk849\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-lpk849, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-lpk849\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-lpk849, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006279c0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-lpk849\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:207]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f832c0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-lpk849\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-lpk849, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-lpk849\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-lpk849, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006279c0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-lpk849\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036404039733743616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T11:25:32Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"69c2fa6a8b272fe7740df4c378e2188c4e2b5b04e9e39368798c5ea62e19215d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036404039733743616","occurred_at":"2026-03-24T11:25:32Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036403373791514624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4539,"pr_state":"closed","pr_sha":"87a94b60840c1bd78c84378b323f1a824af5a7c0","final_merged_sha":"e4ce686fe32b144e5b46173e80928cf3e0dc3359","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T11:22:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036395156256591872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4597,"pr_state":"closed","pr_sha":"ef72c116e286f8088ce71460f5fbfc8238f18cee","final_merged_sha":"bd2f0e6c361bf3b68e8bce0c60fb9c8be4f1300a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T10:50:14Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b70dbca721eb42d32ac602e350883ee59e9ecbbc6796b24b2ad072b2bdbd7c0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036395156256591872","occurred_at":"2026-03-24T10:50:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036392263734530048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4596,"pr_state":"closed","pr_sha":"67bf3e249451c317252c827c70909873b7b8b0e8","final_merged_sha":"67bf3e249451c317252c827c70909873b7b8b0e8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T10:38:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4595,"pr_state":"closed","pr_sha":"9a6bf6ff253d417da5bacba521bf079a5093c092","final_merged_sha":"c367d34f0a8c2c665e52e5644abc435d913d801f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T09:53:55Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"01650a03bc73ad2a8fcd67b8122a0ce0df636ca8d9a7afc3818ba3fe174952dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"33536a8a869a2828e7f63105d30c906931c1c0bcae10445e2f8bf3a9717fdb0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"871fbf2913db30a3edab13ae8cb0e1b4571139a9011d8678afa66bfec0ead439","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001109140\u003e: \n failed to create HCP cluster ea-listnsdk6p, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: failed waiting for cluster=\"ea-listnsdk6p\" in resourcegroup=\"ea-list-2qfkbm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listnsdk6p, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: failed waiting for cluster=\\\"ea-listnsdk6p\\\" in resourcegroup=\\\"ea-list-2qfkbm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ea6a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm\",\n },\n \u003c*fmt.wrapErrors | 0xc001109110\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listnsdk6p\\\" in resourcegroup=\\\"ea-list-2qfkbm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ea6a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001109140\u003e: \n failed to create HCP cluster ea-listnsdk6p, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: failed waiting for cluster=\"ea-listnsdk6p\" in resourcegroup=\"ea-list-2qfkbm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listnsdk6p, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: failed waiting for cluster=\\\"ea-listnsdk6p\\\" in resourcegroup=\\\"ea-list-2qfkbm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ea6a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm\",\n },\n \u003c*fmt.wrapErrors | 0xc001109110\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listnsdk6p\\\" in resourcegroup=\\\"ea-list-2qfkbm\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ea6a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listnsdk6p in resource group ea-list-2qfkbm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"37b53c22cb2e391bb5a0e3519823ecbcd8d65db3b17a6104d3a73a82dfaa1c87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"de09b0e8d27529fd61db9886865ace99795ff3a7cd4cb65ad264850f24f6d3f0","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f11980\u003e: \n failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gbm28q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-gbm28q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010d12a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q\",\n },\n \u003c*fmt.wrapErrors | 0xc000f11950\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-gbm28q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010d12a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:86]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f11980\u003e: \n failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gbm28q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-gbm28q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010d12a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q\",\n },\n \u003c*fmt.wrapErrors | 0xc000f11950\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-gbm28q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010d12a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-gbm28q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"3b7c432a31e28a40980d47ad772473ab354da1384e1ad8c1170c78be02ad08ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"cc17e4e4415b2d185b06d56fed62c7192c7d4a853972145a82c7e8e4c691fd07","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0005dab40\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-55std9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-55std9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015d0c20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9\",\n },\n \u003c*fmt.wrapErrors | 0xc0005dab10\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-55std9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015d0c20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0005dab40\u003e: \n failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-55std9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-55std9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015d0c20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9\",\n },\n \u003c*fmt.wrapErrors | 0xc0005dab10\u003e{\n msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-55std9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015d0c20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-55std9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"44e9b5b2320e1a7650b41933eb3395064fb88f6fcd8c207a77e0a20cc5eed73f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"4d8001544bf3027c69d4b103b4cb068e55f2d6e900be61bd1a24cce1ffbbaab1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"1cc9536775f8c18bc5f893377deb791702d4353289ee457be50b0a0e03a24750","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d82ab0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zmbms2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zmbms2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dc1d40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2\",\n },\n \u003c*fmt.wrapErrors | 0xc000d82a80\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zmbms2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dc1d40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d82ab0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zmbms2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zmbms2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dc1d40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2\",\n },\n \u003c*fmt.wrapErrors | 0xc000d82a80\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zmbms2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dc1d40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zmbms2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"4ed66e64986fe640b9d113224bbb39cfca2b2f2e34f848532b750810da27a8ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:121]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:121]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"5dfaeceabf45acd4cd22c1a4b5df7b740539d07211419e2e6025f1d7b43c6102","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"5f2bc1ad5eaf952e9cbca1cc8ea5c53917f8788590fe82c61ec0eae08bdb4dd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:120]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:120]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"6e449fa0893d082ca9d1d3b7d20703f5d9c406825ba06ea71a9489ffb8d90f3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"bda18407633dfdc78b4a0cdd553258452b4fd0663c29d4361e90531c810115cb","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007a9170\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-ngfjrh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ngfjrh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dd6600\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh\",\n },\n \u003c*fmt.wrapErrors | 0xc0007a9050\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ngfjrh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dd6600\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007a9170\u003e: \n failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-ngfjrh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster sre-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ngfjrh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dd6600\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh\",\n },\n \u003c*fmt.wrapErrors | 0xc0007a9050\u003e{\n msg: \"failed waiting for cluster=\\\"sre-hcp-cluster\\\" in resourcegroup=\\\"admin-api-breakglass-ngfjrh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dd6600\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster sre-hcp-cluster in resource group admin-api-breakglass-ngfjrh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"763cd92c0af36f54568b30a0065456a20080878d36d925d083ac29a721a33f9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"cd7885a3c22cb3ea0abad0f15b6ff2b2a528ca2fe525dcad599e3c39707095cd","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f4ff0\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-pv6phf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-pv6phf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011dab40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf\",\n },\n \u003c*fmt.wrapErrors | 0xc0011f4fc0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-pv6phf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011dab40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f4ff0\u003e: \n failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-pv6phf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-pv6phf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011dab40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf\",\n },\n \u003c*fmt.wrapErrors | 0xc0011f4fc0\u003e{\n msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-pv6phf\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011dab40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-pv6phf\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"7a6582a2bbef961d0623a6f2276c25179d3769f9d5c9f6e0cac6845061fd34ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"cd4eba35666f22b8782ff09138bd5f634b4a8c58077bec431e7dcfa4e672f4d2","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00109efc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8wnpmk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8wnpmk, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-8wnpmk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8wnpmk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcc9b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8wnpmk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00109efc0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8wnpmk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8wnpmk, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-8wnpmk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8wnpmk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcc9b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8wnpmk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"85d5a89245d074e3ae8dd15baec61444509419d43050cb0773779b916c20f6a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"1ccd1a7335a5eb1adcf4823b75f78d29df681a2bae6aa24cd9495b1d9236fe1d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006b6f30\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2s59zq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s59zq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fddd80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq\",\n },\n \u003c*fmt.wrapErrors | 0xc0006b6f00\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s59zq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fddd80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006b6f30\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2s59zq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s59zq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fddd80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq\",\n },\n \u003c*fmt.wrapErrors | 0xc0006b6f00\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-2s59zq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fddd80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-2s59zq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"875c5a084b5cf36b89cec3518299464a783427750afb23dd1d7d5df1250ad02f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"d27c4e662c837a33e603b548a107cbc51bf3d23340383c45f3060b43b00349bd","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eb7200\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-slq2bs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-slq2bs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bf640\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs\",\n },\n \u003c*fmt.wrapErrors | 0xc000eb71d0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-slq2bs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bf640\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eb7200\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-slq2bs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-slq2bs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bf640\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs\",\n },\n \u003c*fmt.wrapErrors | 0xc000eb71d0\u003e{\n msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-slq2bs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010bf640\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-slq2bs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"8820c09feac41d86254a175d8d1b95efd48b401498f9be5e581af4d79cfc2ad7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"0391b9ffffb952b54eb6572fcc40310e0a458591fa9e3183b636b56ebfa4f23b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:115]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e7e9f0\u003e: \n failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-blsl7w\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-blsl7w\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d9d940\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w\",\n },\n \u003c*fmt.wrapErrors | 0xc000e7e9c0\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-blsl7w\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d9d940\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:115]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e7e9f0\u003e: \n failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-blsl7w\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-blsl7w\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d9d940\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w\",\n },\n \u003c*fmt.wrapErrors | 0xc000e7e9c0\u003e{\n msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-blsl7w\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d9d940\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-blsl7w\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"8b10670ebd351c05cd5de747ac19927f7706c4c90c0c461f1feb8ee5c08dec22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"1cf0bac0c46853d7f86fa74fee48a9603ffba3e98458532cfe51107f204ecba9","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001129500\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-msj7rl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-msj7rl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e85520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl\",\n },\n \u003c*fmt.wrapErrors | 0xc0011294d0\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-msj7rl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e85520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001129500\u003e: \n failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-msj7rl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-msj7rl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e85520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl\",\n },\n \u003c*fmt.wrapErrors | 0xc0011294d0\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-msj7rl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e85520\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-msj7rl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"a5315c919ca1a35b31965107c7353c360ae36fe600090ecfdd968b18f7dc396b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"a6fdd99cb343ca5cf69b18f3ac3cd8014af64e37177cb70f9064f571a1dd6252","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"6ecef520616a6c4eab9396dab716ec3383e58adb792e6cd64ad834a30837dc10","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001403410\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bd54jc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bd54jc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00101f780\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc\",\n },\n \u003c*fmt.wrapErrors | 0xc0014033e0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bd54jc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00101f780\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001403410\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-bd54jc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bd54jc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00101f780\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc\",\n },\n \u003c*fmt.wrapErrors | 0xc0014033e0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-bd54jc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00101f780\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-bd54jc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"aa98ee191ff3e2a65d651d8e7c924c6acc3b01d804b7ac812b5938a22f5ce8a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"80a5cd94465ec8b5dcd167cc81dc80b4859f7cdba2d4b71047654e4d57015aa4","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009568a0\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-52t5m6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-52t5m6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fd7090\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6\",\n },\n \u003c*fmt.wrapErrors | 0xc000956870\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-52t5m6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fd7090\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009568a0\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-52t5m6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-52t5m6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fd7090\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6\",\n },\n \u003c*fmt.wrapErrors | 0xc000956870\u003e{\n msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-52t5m6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fd7090\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-52t5m6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"bc7ae5dca84a04ef9586450962c2082c71af52f0520b05bce71458416982ae56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"bcc9fee9a2aa375aca45df91be9951ca246ac5000900c38beb59011f5661516a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"8799e33e081f98d838b656fecfaf6850be6b34f3ddc85859f08ba7c0156d6276","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e0a8d0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5qzmjb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5qzmjb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc3150\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb\",\n },\n \u003c*fmt.wrapErrors | 0xc000e0a8a0\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5qzmjb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc3150\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:82]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e0a8d0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-5qzmjb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-update-nodes-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5qzmjb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc3150\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb\",\n },\n \u003c*fmt.wrapErrors | 0xc000e0a8a0\u003e{\n msg: \"failed waiting for cluster=\\\"np-update-nodes-hcp-cluster\\\" in resourcegroup=\\\"nodepool-update-nodes-5qzmjb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fc3150\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5qzmjb\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"d50f29130b4ad92915975c08451ef6f42df6374fc4d0ba3a1c11224e7016d83d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"9accada4f09861e8dfab9e799612abc41eee09769b0fd6f3944dd4afa9cac215","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014ce9c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-s4lrjc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-s4lrjc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011c0190\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc\",\n },\n \u003c*fmt.wrapErrors | 0xc0014ce990\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-s4lrjc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011c0190\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014ce9c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-s4lrjc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-s4lrjc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011c0190\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc\",\n },\n \u003c*fmt.wrapErrors | 0xc0014ce990\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-s4lrjc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011c0190\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-s4lrjc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"d970bd4a07d812826d7e31f6b60d74037cff1f0812e0df0a23de835684df57d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"2cae3607b088db28226ac72412a7e5ab78d67f73575931637233e6d484c9f19b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007feea0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4cwxb8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-4cwxb8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00126cb10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8\",\n },\n \u003c*fmt.wrapErrors | 0xc0007fee70\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-4cwxb8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00126cb10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007feea0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4cwxb8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-4cwxb8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00126cb10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8\",\n },\n \u003c*fmt.wrapErrors | 0xc0007fee70\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-4cwxb8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00126cb10\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4cwxb8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"e5f953e026d464b8cb8f87f05cef2a829078312823e0b4abdc5cdd30b79b7959","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"94f4ae2b2661fe164119cd7b68561bc30a9c0a5adc4270b820059575a0f794d7","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00092f350\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-v9djvs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-v9djvs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001179bf0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs\",\n },\n \u003c*fmt.wrapErrors | 0xc00092f320\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-v9djvs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001179bf0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00092f350\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-v9djvs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-v9djvs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001179bf0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs\",\n },\n \u003c*fmt.wrapErrors | 0xc00092f320\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-v9djvs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001179bf0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-v9djvs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"f11c1e1db4bcb2b03511199c47f3c998854894c27b43cc49a7a3de62e1dbe87f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380982692548608","occurred_at":"2026-03-24T09:53:55Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}}],"failure_pattern_summary":{"signature_count":19,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380284345126912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"9de7f0cacb1b52ccd637ce9ccd3959ca283d85ca","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T09:51:08Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"11f3dc5674ccee4d6ef4d2b6921ce00e017071fded25b9902ca9fbfdcf40129c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380284345126912","occurred_at":"2026-03-24T09:51:08Z","failed_at":"e2e","signature_id":"90fc0a7d892fd7517cc0dfdeb0326052ddcf8be0fabb316629a9f9e4bc6c3fdb","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00124ee10\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-dp5nl9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011be660\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9\",\n },\n \u003c*fmt.wrapErrors | 0xc00124ede0\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-dp5nl9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011be660\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00124ee10\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-dp5nl9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011be660\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9\",\n },\n \u003c*fmt.wrapErrors | 0xc00124ede0\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-dp5nl9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011be660\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}},{"row_id":"93ce85ed47141dd3adb21e7ddf3b7ae54692eb212d43b427c6a0e51f604fc8ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036380284345126912","occurred_at":"2026-03-24T09:51:08Z","failed_at":"e2e","signature_id":"3b949e86a9ce2c98afc9d29428188f7e5ccb4e79c76186720a2e85306242f54c","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e7c7e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mx72rw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mx72rw, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-mx72rw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mx72rw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c2170\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mx72rw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e7c7e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mx72rw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mx72rw, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-mx72rw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mx72rw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c2170\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mx72rw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036372738683179008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4442,"pr_state":"closed","pr_sha":"8ac2a500d7cade2ccbab11984c86c49cd0e20d4f","final_merged_sha":"8ac2a500d7cade2ccbab11984c86c49cd0e20d4f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T09:21:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036362939249725440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4583,"pr_state":"closed","pr_sha":"39d00655b0878452b6e129a6301b0f285e262876","final_merged_sha":"39d00655b0878452b6e129a6301b0f285e262876","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T08:42:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4594/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036360912859828224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4594,"pr_state":"closed","pr_sha":"80b2e3ca8db6b4fd68929cdc0e76275e0c2788df","final_merged_sha":"80b2e3ca8db6b4fd68929cdc0e76275e0c2788df","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T08:34:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036352354697089024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4554,"pr_state":"closed","pr_sha":"1ba8557e7aa3030dfdb65d2dedde1be704afe79f","final_merged_sha":"cab9df44f4aaadfa6301e36bed1fb8403e84a43b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T08:00:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036352223352459264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"047f4394fbfb3ee76f54b58c25ba5019edf9b456","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T07:59:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036350810824118273","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4593,"pr_state":"closed","pr_sha":"162efb4b45d2fa40637f6b5f8cdd17b89ca71cb4","final_merged_sha":"162efb4b45d2fa40637f6b5f8cdd17b89ca71cb4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T07:54:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036347706821054464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4592,"pr_state":"closed","pr_sha":"25d2db7ef04240bfc15c474bf8388ea789501a21","final_merged_sha":"4bcc7ca5dce31f4cecce91af4a237c05cecd7237","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T07:41:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036346654000418816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4585,"pr_state":"closed","pr_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","final_merged_sha":"5ed7b324786bb326a526a06fec3e2fdc2ceec01f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T07:37:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036346504947437568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4583,"pr_state":"closed","pr_sha":"5f59c6e2d1709c4e9fbc7ed6e536d7e592250dad","final_merged_sha":"39d00655b0878452b6e129a6301b0f285e262876","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T07:36:54Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"23afafac0ee99aed7802ba4062a41c163de42e014569a541572bdce1d7e6df01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036346504947437568","occurred_at":"2026-03-24T07:36:54Z","failed_at":"provision","signature_id":"e243e269102704c951785ba58cc40912c3a06dc3133fbca624851e1d2e3ea56f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/arobit","test_suite":"step graph","failure_text":"time=2026-03-24T08:14:37.126Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-03-24T08:14:37.623Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:prow-j7437568-mgmt-1 clusterType:mgmt dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:33bed43b09e3d2384c7661c74f6ed79ca53b0c25a3e28d77f139a7b9d50fbf1a pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit]] geoShortId:us kusto:map[enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net serviceLogsDatabase:ServiceLogs] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan: clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-prow-j7437568-mgmt-1 rpAccountName:placeholder rpNamespace: rpSan: rpSecretName:rplogs] image:map[digest:sha256:2ae5282c99fa8c82bb394b23dbcb9ea0fa553e4ec3cce9e39d110f44b6282062 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5c0d020-14b6-44fa-9911-d456e0602838 region:westus3 secretProvider:map[keyVault:ah-prow-mg-j7437568-1] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-24T08:14:37.623Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-24T08:14:37.957Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit namespace=arobit\ntime=2026-03-24T08:14:38.239Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit dryRun=false\ntime=2026-03-24T08:14:38.239Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-24T08:14:38.370Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-24T08:14:42.079Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit err=\"failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema\"\ntime=2026-03-24T08:14:42.079Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-24T08:14:42.079Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-24T08:14:42.703Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit release=arobit namespace=arobit status=failed description=\"Release \\\"arobit\\\" failed: failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema\" values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:prow-j7437568-mgmt-1 clusterType:mgmt dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:33bed43b09e3d2384c7661c74f6ed79ca53b0c25a3e28d77f139a7b9d50fbf1a pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit]] geoShortId:us kusto:map[enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net serviceLogsDatabase:ServiceLogs] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan: clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-prow-j7437568-mgmt-1 rpAccountName:placeholder rpNamespace: rpSan: rpSecretName:rplogs] image:map[digest:sha256:2ae5282c99fa8c82bb394b23dbcb9ea0fa553e4ec3cce9e39d110f44b6282062 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5c0d020-14b6-44fa-9911-d456e0602838 region:westus3 secretProvider:map[keyVault:ah-prow-mg-j7437568-1] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-03-24T08:14:42.703Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yRQU/qQBDH7/0Ukz3RpBBeIS+EFw5P48ngQT1hPGx3h7rQzjSzU4gJH95UUIhR4frb338yO/+kQgXByK04jDADb9WqLSrscbFCp7eB/DSqBCoz2KM7W+Mnwg3SO4mNdR84fUoAAMwDyiY4/O8ct6QmA2OFi6D9JcvWikc5MpPtM9dMy1DObXOhXrVRUe65wp8C36hXgXyg8lzisP9lmxzkOVNQljOZ5Plfsm4LFELFeNMdMSY72L6gIGioMaqtGyhQt4gEPW8VO9wz+TD/2x+O+vn4cTiZ/hlPR5OFSWEwgF+ccb4waZrsYMWBYB3IzwIRykn1THBs/LTqrx0nO2iEu9fjohk07PfyZUMyqLjMoMYYbdmN5O5AULyefN5jdG8BAAD//yumHLugAgAA\"\ntime=2026-03-24T08:14:42.703Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit\ntime=2026-03-24T08:14:42.703Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b3054742c32446c91766fddbabfa349e00ce0ae344f5050d657f5db6e29b1","failure_pattern":"failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\"fluentbit\"].limits: field not declared in schema","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to create typed patch object (arobit/arobit-forwarder; apps/v1, Kind=DaemonSet): .spec.template.spec.containers[name=\\\"fluentbit\\\"].limits: field not declared in schema"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1a6b3054742c32446c91766fddbabfa349e00ce0ae344f5050d657f5db6e29b1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036346343269601280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4234,"pr_state":"closed","pr_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","final_merged_sha":"ac3354ace12fb48d60cedc2365d42cdcd0e45759","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-24T07:36:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"117b640ccd8500ff54562050f00563e2376d9bcbae1e1d23654c61f39261d8f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036346343269601280","occurred_at":"2026-03-24T07:36:16Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-24T07:57:26.824Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-24T07:57:29.113Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-24T07:57:30.737Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=8bffa6a59c30c76b6c0f52143355fc7794894833c46ae7842be2f2600ca890c4\ntime=2026-03-24T08:09:03.105Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9601280-svc/providers/Microsoft.Resources/deployments/8bffa6a59c30c76b6c0f52143355fc7794894833c46ae7842be2f2600ca890c4/operationStatuses/08584272678355634411\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9601280/providers/Microsoft.Resources/deployments/rp_cosmos_db\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"FailedIdentityOperation\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9601280/providers/Microsoft.DocumentDB/databaseAccounts/arohcpprow-rp-j9601280' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\\\\\\\\\\\\\"error\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"BadRequest\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9601280-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/admin-api' was not found.\\\\\\\\\\\\\\\"}}'.'.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e7fca522141975e7bca69490a839ce35e4cd1ce022717bb9c58f7125b869988f","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/admin-api' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e7fca522141975e7bca69490a839ce35e4cd1ce022717bb9c58f7125b869988f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036345797041197056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"89b4c84a825d0daa360e075b90f97d33b25c964c","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T07:34:06Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"4463ea0fb59936da4eabcb11ebb3341acc464c8af7f56971a2d063b60ec75e83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036345797041197056","occurred_at":"2026-03-24T07:34:06Z","failed_at":"e2e","signature_id":"c620672e35d0e22e64e60e200147edbf569212cf1db15225f39a007b01d242c2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00141a4e0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-pxhs5d\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00086bb00\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00141a4e0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-pxhs5d\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00086bb00\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pxhs5d\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-pxhs5d, error: context deadline exceeded"}},{"row_id":"ca1b6973fee89a815f7963406a5830efa394132b63350587a14ae3b43698300a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036345797041197056","occurred_at":"2026-03-24T07:34:06Z","failed_at":"e2e","signature_id":"d3a76eba9eace9dc0988fc44e852ca88d0bc5e809d62079b3fa508e9dcf22149","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009b5440\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5fwbrc\": PUT https://rp.j1197056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fwbrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fwbrc\\\": PUT https://rp.j1197056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fwbrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009b5420\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fwbrc\\\": PUT https://rp.j1197056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fwbrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001511440\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"206\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"502f5de0-2332-46f7-b52d-c5c3ef55b417\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 08:19:01 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009b4600\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 206,\n TransferEncoding: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009b5440\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-5fwbrc\": PUT https://rp.j1197056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fwbrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fwbrc\\\": PUT https://rp.j1197056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fwbrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0009b5420\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-5fwbrc\\\": PUT https://rp.j1197056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-5fwbrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001511440\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"206\"],\n \"X-Envoy-Upstream-Service-Time\": [\"40\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"502f5de0-2332-46f7-b52d-c5c3ef55b417\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 08:19:01 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009b4600\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 206,\n TransferEncoding: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fc29a12be19e8b3bf64c6f970786e4d8cab16ee0ffab8d077a4d10deb989fed8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036345797041197056","occurred_at":"2026-03-24T07:34:06Z","failed_at":"e2e","signature_id":"a80add862b76fde0be7bf62cc57840c5a28189932232ebdd598ac773abe02fa3","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ea49f0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-s94fkv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-s94fkv, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-s94fkv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-s94fkv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0017d8870\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-s94fkv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ea49f0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-s94fkv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-s94fkv, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-s94fkv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-s94fkv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0017d8870\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-s94fkv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036342860818681856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4576,"pr_state":"closed","pr_sha":"4eda0032750666275592116643efcf64bdf1b330","final_merged_sha":"4eda0032750666275592116643efcf64bdf1b330","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T07:22:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036333032184483840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4592,"pr_state":"closed","pr_sha":"25d2db7ef04240bfc15c474bf8388ea789501a21","final_merged_sha":"4bcc7ca5dce31f4cecce91af4a237c05cecd7237","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T06:43:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fe5ad84d490a60286799ac8dd35f2237ebcc15c69adbf10a0b96ceca1d686933","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036333032184483840","occurred_at":"2026-03-24T06:43:22Z","failed_at":"provision","signature_id":"ff880cfb138e240d4139e64886fbd41eea79777c725e5bc81c662e5a060e45de","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-24T07:20:31.972Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-24T07:20:32.576Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj4483840.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj4483840.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:8f3a4d06-1d40-40ff-aca0-cb33dc4c01f1 databaseName:arohcpprow-rp-j4483840 databaseUrl:https://arohcpprow-rp-j4483840.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-kr5iz86v/pipeline@sha256:bac45a637794cb0862fcf1c22b4d7b6260af4f04cf3ed83cfc787e674f96d8fd replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-kr5iz86v/pipeline:pull workloadIdentityClientId:34f2105a-cd04-4a7f-8c10-cbab7c9fdabf workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:8f3a4d06-1d40-40ff-aca0-cb33dc4c01f1 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-24T07:20:32.576Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-24T07:20:32.848Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-24T07:20:33.195Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-24T07:20:33.195Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-24T07:20:33.345Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-24T07:25:42.434Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-24T07:25:42.434Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-24T07:25:42.434Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-24T07:25:46.647Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj4483840.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj4483840.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:8f3a4d06-1d40-40ff-aca0-cb33dc4c01f1 databaseName:arohcpprow-rp-j4483840 databaseUrl:https://arohcpprow-rp-j4483840.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-kr5iz86v/pipeline@sha256:bac45a637794cb0862fcf1c22b4d7b6260af4f04cf3ed83cfc787e674f96d8fd replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-kr5iz86v/pipeline:pull workloadIdentityClientId:34f2105a-cd04-4a7f-8c10-cbab7c9fdabf workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:8f3a4d06-1d40-40ff-aca0-cb33dc4c01f1 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-24T07:25:46.648Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQW/aQBCF7/4VI59A8kYI0laiyoGmPVWpUNNTqh7Wu6+wYb1jza5BRPz4yrgEKzIR17fvm30zmsk8EgkiN2IQ6Y6sTjrp0mPE5TNM+u6CncckLqwK6qQfusKrhC3CUYm1Nid5/DsjIsoXRpaN919csC6s8oLyUpsNglV1470qz7oWVmtT50UH/mSPAcpDW4iCh0mOwwD4CNk6g96LOsEVkjgTB6CvqD3vK4Q0wA34F01as7gX3YZYsndmf3R5z7vTN+/gS0DaEgjJGf2mkSviPsII0lJ46yzk3usY+0OK3QjUBnu11Y1Pl8f0wMElluu6/s8sjOGmm9Rl7z2Hv271oOt+MP3SCJQ0IbkKyhwt17IX3e2mtPI7q5H9+ZxtmhISkBC/tQsbswPt1hBQmyUmXdVUIu2AQCOrE1p5lE8n049qMlPT21+TT/PpZD6bPeVjurmhi54P89vpUz4eZwd6Zhdo44K9cyFAemfGgc7X1T+rt/eUHagWbl/PQQuq2Xbm64oU5HlVUIUY9aotyWIhVO57zVtE8y8AAP//0PWTcQwEAAA=\"\ntime=2026-03-24T07:25:46.654Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-24T07:25:46.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036296044051959808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4574,"pr_state":"closed","pr_sha":"162cc07eb137e2c6e706c814b9d10c4449bc72e0","final_merged_sha":"162cc07eb137e2c6e706c814b9d10c4449bc72e0","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T04:16:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036272963430387712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4587,"pr_state":"closed","pr_sha":"49f81d94ae91271fb6f2b5b819c67709937251ba","final_merged_sha":"49f81d94ae91271fb6f2b5b819c67709937251ba","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T02:44:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036269017831837696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4589,"pr_state":"closed","pr_sha":"288e1d5f3d185b033d8cb1bbd8dc5041eaa74c1e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T02:29:00Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5b675d4471830aaeb2774f7ddf03ec6e06188fabaffb26499678a3531ed2db91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036269017831837696","occurred_at":"2026-03-24T02:29:00Z","failed_at":"provision","signature_id":"eb42f536566c06969e79bb4dd530b90fade97f988fe3d18503444c2160414c7b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/svc-mgmt-permissions","test_suite":"step graph","failure_text":"time=2026-03-24T03:04:07.422Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description=\"Step svc-mgmt-permissions\\n Kind: ARM\\n Template: templates/svc-mgmt-permissions.bicep\\n Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam\"\ntime=2026-03-24T03:04:07.900Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions\ntime=2026-03-24T03:04:08.871Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=6f5dc51387bf3d351f1b8e5546f891a1318b5147b96d6ce634f7f9b9588dd09a\ntime=2026-03-24T03:04:39.110Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1837696-mgmt-1/providers/Microsoft.Resources/deployments/6f5dc51387bf3d351f1b8e5546f891a1318b5147b96d6ce634f7f9b9588dd09a/operationStatuses/08584272854372039071\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentExists\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The role assignment already exists. The ID of the existing role assignment is 0890b582518a5466b9bb14be0e9debd7.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentExists; detail message The role assignment already exists. The ID of the existing role assignment is \u003chex\u003e.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2c08b2b0f48b1745f538e5727c28908ad9a04645767c115dc1cc38892e968435"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036257137084076032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"c86c5ea8623e834f6577d715795f8cf706e04c3f","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-24T01:41:47Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"b27b51e1d20c3c81e38716842d8accb73ec3b726884c2e59893e079afe66802d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036257137084076032","occurred_at":"2026-03-24T01:41:47Z","failed_at":"e2e","signature_id":"1b87ba7d437a89ba0a8ac606a09b15f214ec2ec2eaea4bca2ef8d82ed62ba778","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e07c20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ftgghp\": PUT https://rp.j4076032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ftgghp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-ftgghp\\\": PUT https://rp.j4076032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ftgghp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e07c00\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-ftgghp\\\": PUT https://rp.j4076032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ftgghp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011a7080\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"206\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"2542e3fe-ba0f-458c-8e87-13393388e6f3\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 02:30:15 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e07b20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 206,\n TransferEncoding: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:122]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e07c20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ftgghp\": PUT https://rp.j4076032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ftgghp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-ftgghp\\\": PUT https://rp.j4076032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ftgghp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e07c00\u003e{\n msg: \"failed starting cluster creation \\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-ftgghp\\\": PUT https://rp.j4076032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ftgghp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Required value: field cannot be null\\\",\\n \\\"target\\\": \\\"properties.etcd.dataEncryption.customerManaged.kms.visibility\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011a7080\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Length\": [\"206\"],\n \"X-Envoy-Upstream-Service-Time\": [\"34\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"2542e3fe-ba0f-458c-8e87-13393388e6f3\",\n ],\n \"Date\": [\n \"Tue, 24 Mar 2026 02:30:15 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e07b20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 206,\n TransferEncoding: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ed78204c28af6f799fb40da2e39ff336f48cc637ad4ce7310e253734a7d6f005","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036257137084076032","occurred_at":"2026-03-24T01:41:47Z","failed_at":"e2e","signature_id":"4f871aedb2d0e84b8c54e1349e26e7d542fb83df144af51bd450702870506eba","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d54a20\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bhmmpv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-bhmmpv, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-bhmmpv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-bhmmpv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001376510\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-bhmmpv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d54a20\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bhmmpv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-bhmmpv, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-bhmmpv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-bhmmpv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001376510\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-bhmmpv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dp5nl9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dp5nl9, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036255031828680704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4463,"pr_state":"open","pr_sha":"8bf53059836d7c658117add712660a10ebbab48b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-24T01:33:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036240993761628160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4587,"pr_state":"closed","pr_sha":"49f81d94ae91271fb6f2b5b819c67709937251ba","final_merged_sha":"49f81d94ae91271fb6f2b5b819c67709937251ba","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T00:37:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036236974498516992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4584,"pr_state":"closed","pr_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","final_merged_sha":"4999eef2bb460e178d184cded32a17edd3e2ffea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-24T00:21:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}