{"meta":{"date":"2026-08-21","timezone":"UTC","generated_at":"2026-08-25T21:18:08Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-21T23:00:43Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"13893233c711c4a654559a02d7006327fbe41529afa9576d2a445c21d24b534c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"e3efa032fccb6cd516133bd1ad89ebb60e8dae5741cd61caf0ccbc55f84ea58e","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-62zzff\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102a5c0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-62zzff\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-stlzppcpw4zq\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n \"name\": \"1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:44:09.672524927Z\",\n \"endTime\": \"2026-08-21T23:44:17.756539054Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-62zzff\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102a5c0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-62zzff\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-stlzppcpw4zq\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n \"name\": \"1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:44:09.672524927Z\",\n \"endTime\": \"2026-08-21T23:44:17.756539054Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6e0339db7bd54e6d51e0a052774819897d4aae43976b41e9f3679271ea556d4f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"150b0bb76d495944c6d661fced1c35ea9eba64f6677f307eda69bafd37ba9bb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"cae38e430d8e80e3a74035572e24eda7a12fa1071db2d9d61da01032369a656b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ltpqhc\" to 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00071a3e0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ltpqhc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c4ngqczk4ph5\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n \"name\": \"80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:42:45.54327206Z\",\n \"endTime\": \"2026-08-21T23:42:47.666030908Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ltpqhc\" to 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00071a3e0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ltpqhc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c4ngqczk4ph5\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n \"name\": \"80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:42:45.54327206Z\",\n \"endTime\": \"2026-08-21T23:42:47.666030908Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"220c303bbf477a74273991fde7c9a5f336a0663963edfed72cb63f983e8df885","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"160c44afa7307c45b06ec93c5a18b256d5d98dd0cc533e5295625a9cd8260c94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"c65bf0079bcd2ce58e12f45cf9f37bbe8f148f64187938cc3ba61a042251c88d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f0d698\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f0d698\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"4935294a4f1476e5013a12e5172b36eda7d233362183d0e151d5ec5fa85414ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"6ce3f21db4e03dbc02da614b27bd974fe76c85c6dcbc700f959708a7547d5026","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-7jwrlb\" to 4.23.0-0.nightly-multi-2026-08-21-150158\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00071fec0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-7jwrlb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ddtbb45g74fw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n \"name\": \"47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:45:19.683338639Z\",\n \"endTime\": \"2026-08-21T23:45:20.631024642Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-7jwrlb\" to 4.23.0-0.nightly-multi-2026-08-21-150158\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00071fec0\u003e: \n failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-7jwrlb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ddtbb45g74fw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n \"name\": \"47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:45:19.683338639Z\",\n \"endTime\": \"2026-08-21T23:45:20.631024642Z\",\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f9bfb1faf51b51475c30265bfb24f836c9386a47bad2413a3d8e330ed7c9045e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"97e4bcd4db79fa32f5455470afbb933c5ef7df965e9d2ac97e7e1d54a0e66482","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"542d678565bcb4bd14e18783f6cd563900f9c155e36aa7035ac1070cce39883f","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0023345d0\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-57d8dvdfgh9q, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-57d8dvdfgh9q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-57d8dvdfgh9q, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0023345d0\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-57d8dvdfgh9q, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-57d8dvdfgh9q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-57d8dvdfgh9q, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-q7rqmb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: failed waiting for cluster=\"cluster-candidate-5-0-q7rqmb\" in resourcegroup=\"rg-candidate-5-0-w7qwtc6pmx48\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: context deadline exceeded"}},{"row_id":"cd0ce3b9be04d52c774db0e5b2e37af7798253e97594388baabfc7a635bfd0a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"f9145ac5e2884f8eb9ccef988d1f85c4fe55e8c0ff9c9f95d10d84fcc0da9011","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d91330\u003e: \n node pool upgrade verification failed: t6b8h6h3q5f1q8x-nps-420-422-tm8gq-d7mjd (no version in containerRuntimeVersion \"cri-o://1.35.4\"); t6b8h6h3q5f1q8x-nps-420-422-tm8gq-zqts7 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"t6b8h6h3q5f1q8x-nps-420-422-tm8gq-d7mjd\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"t6b8h6h3q5f1q8x-nps-420-422-tm8gq-zqts7\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9463da4154ce4a2354a72c8ac8d2b373ea30e73f8881235d77e00f3c4642f31e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:df3ebe9aea9dcc90e0e472cafb478cb56fcca54b6aa51326c3af1afdca3ab557\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d91330\u003e: \n node pool upgrade verification failed: t6b8h6h3q5f1q8x-nps-420-422-tm8gq-d7mjd (no version in containerRuntimeVersion \"cri-o://1.35.4\"); t6b8h6h3q5f1q8x-nps-420-422-tm8gq-zqts7 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"t6b8h6h3q5f1q8x-nps-420-422-tm8gq-d7mjd\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"t6b8h6h3q5f1q8x-nps-420-422-tm8gq-zqts7\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9463da4154ce4a2354a72c8ac8d2b373ea30e73f8881235d77e00f3c4642f31e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:df3ebe9aea9dcc90e0e472cafb478cb56fcca54b6aa51326c3af1afdca3ab557\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: t6b8h6h3q5f1q8x-nps-420-422-tm8gq-d7mjd (no version in containerRuntimeVersion \"cri-o://1"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","220c303bbf477a74273991fde7c9a5f336a0663963edfed72cb63f983e8df885","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","6e0339db7bd54e6d51e0a052774819897d4aae43976b41e9f3679271ea556d4f","f9bfb1faf51b51475c30265bfb24f836c9386a47bad2413a3d8e330ed7c9045e"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-21T02:00:08Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"23f6d34bb24f4521e6569a71c2209979f397a5c7354d92a388c676b39ca756ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"6b18799e0c02120baa6b7406c38ed69946bfb8f075f9525de85fb5d61e35a451","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-7kjkxp\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001506ab0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rncmtdm68xg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-7kjkxp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-7kjkxp\" to minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001506ab0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rncmtdm68xg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-7kjkxp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"816d4adf1b0cead0de869e12fb780cc1877997e002f751655ae6980e24a728c2","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"471fd0c83e2378ba771f9ebca0a3902f87df1cde53027572b1dc25fa691553cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"53444ab8124f99545024afe8c22d658c5922fe471870c6bfb72c71ce7c7d2fd9","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706700\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7ts9cp\" in resourcegroup=\"rg-np-version-upgrade-7ts9cp-bztnq2hvf6sh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n \"name\": \"6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:54:35.924673301Z\",\n \"endTime\": \"2026-08-21T03:13:35.571507889Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706700\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7ts9cp\" in resourcegroup=\"rg-np-version-upgrade-7ts9cp-bztnq2hvf6sh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n \"name\": \"6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:54:35.924673301Z\",\n \"endTime\": \"2026-08-21T03:13:35.571507889Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5bbafb9300511a466b9a17fc3cdfbb0a87e844630a1573299e19d9f76baaec7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"0b9c27a850dd92839570250cc1ea4c147b359cb2854e15193a15f04b24871364","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-w7qwtc6pmx48/cluster-candidate-5-0-q7rqmb should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0012e43f0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-q7rqmb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: failed waiting for cluster=\"cluster-candidate-5-0-q7rqmb\" in resourcegroup=\"rg-candidate-5-0-w7qwtc6pmx48\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-w7qwtc6pmx48/cluster-candidate-5-0-q7rqmb should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0012e43f0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-q7rqmb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: failed waiting for cluster=\"cluster-candidate-5-0-q7rqmb\" in resourcegroup=\"rg-candidate-5-0-w7qwtc6pmx48\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-q7rqmb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: failed waiting for cluster=\"cluster-candidate-5-0-q7rqmb\" in resourcegroup=\"rg-candidate-5-0-w7qwtc6pmx48\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-q7rqmb in resource group rg-candidate-5-0-w7qwtc6pmx48, error: context deadline exceeded"}},{"row_id":"754a830c3af960f419cdca5ba87b94c085c27183a95247764e3533a568f62970","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"34afd7909da6ce5cf526eb6e72eebbe525ca7ddbcc6a4275c8538eb2c2fc8879","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088a60\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9pxxtrlstzqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n \"name\": \"69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:06:00.903480575Z\",\n \"endTime\": \"2026-08-21T02:25:03.149034019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001088a60\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9pxxtrlstzqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n \"name\": \"69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:06:00.903480575Z\",\n \"endTime\": \"2026-08-21T02:25:03.149034019Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9682a44e2ab83966f233362577a4aeb0ce3e5c3c0c44500fbc2ad32a1c222a2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"e2e437dfbe2809e1622aea345f8304dc17ed03f4c4745b55ee016489aa7dc065","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-9hq8tp\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000c1fa10\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dltt2s2c9z4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9hq8tp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-9hq8tp\" to minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000c1fa10\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dltt2s2c9z4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9hq8tp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4afd6a1ada377507e7e7e6c2a964127b8acdde371be3fb4961b6586bdd4a43de","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"abf4be7ef2b51eb689797ed142f1f60639c67343deac65fdf65df6911b573d45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"b0896529db37c90b279c53d6fc422ea9199968c1d2dd406897213a62eda70455","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-d9vwt2\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001576c60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bmvjhbv2q2z7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-d9vwt2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-d9vwt2\" to minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001576c60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bmvjhbv2q2z7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-d9vwt2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n \"target\": \"properties.version.id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n \"target\": \"properties.version.id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"41121f687ee586823386c215e67a10fa5622aef7585d34e6556590734bdccec3","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","41121f687ee586823386c215e67a10fa5622aef7585d34e6556590734bdccec3","4afd6a1ada377507e7e7e6c2a964127b8acdde371be3fb4961b6586bdd4a43de","67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","816d4adf1b0cead0de869e12fb780cc1877997e002f751655ae6980e24a728c2"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}