{"meta":{"date":"2026-08-22","timezone":"UTC","generated_at":"2026-08-25T21:15:16Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-22T23:00:44Z"},"failed_at":["e2e"],"failed_test_count":18,"occurrences":[{"row_id":"12c7ab6b615e1d33736e09460112f661d71c69d4a9db8777aecd15c281310bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"bd404f1ba751bc9877a1e8fdc24c7640aff9f6d2b3d0bdcdce835db61939593c","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00113fc20\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00113fc20\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"218bbebb6f4176abc4aee719d5c9312ca4a0199d33ace731a2c48d274073fcee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"444e0b46276d6b06038a0349a8a8b1b9c6bd0d97e65d53c08b2f7737e27ce63b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001039770\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-21-221014, targetMinor=4.23.0-0.nightly-multi-2026-08-22-063717) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-22-063717\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001039770\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-21-221014, targetMinor=4.23.0-0.nightly-multi-2026-08-22-063717) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-22-063717\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"3e962435f78cee3abe44643c5d880daadeeaf35d3ede0cd549a916b483c1755b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"5b0f787f537c20118ee76f9fc63fd06eb58ee0eee7e5ee4575316f4340472c37","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc00133a348\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-xcq7z.apps.aro.h4w2d4y2i6t7t2l.p1zq.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc00133a348\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-xcq7z.apps.aro.h4w2d4y2i6t7t2l.p1zq.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"42a00548df9905e7e2ab13e834fcc7206f968a7276c7254dde1c727c3e237c9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"87802490969f4a6d1eb8eb8df9ca6f5f32bcb5b82112d8d6b1e304b1d96e3e03","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f7ed50\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-22-032601, targetMinor=4.21.0-0.nightly-multi-2026-08-22-094635) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-22-094635\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f7ed50\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-22-032601, targetMinor=4.21.0-0.nightly-multi-2026-08-22-094635) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-22-094635\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"5de1cc5b5de5f451f74b229594a4ff7c7ea6eacaf2fc9ce8efa5dc9f37eb96cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"473ae6affbd1614b127350a684f5c57cf48befc5b71ca9167deb53482d16e04e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-21-2pl9bt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbcba0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-6bsfj.apps.aro.i3g1d3x3q2y3h5a.89gn.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-21-2pl9bt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbcba0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-6bsfj.apps.aro.i3g1d3x3q2y3h5a.89gn.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"5fc7dd7b37b99d751df27342872be006db44a7f1647289ed74981bb764d901f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"7ffc802cb180f4209a0fa858155687cc6e2899d3764fe7d6bd1321ae4fd55e84","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-20-jvxvkc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db8420\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5sh42.apps.aro.h9m9l7u5u0u0g0a.fj4n.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-20-jvxvkc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db8420\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5sh42.apps.aro.h9m9l7u5u0u0g0a.fj4n.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"723e9f0dae55935bf4e518e448afeb64f1f1485af51c52e98bbbbe1382f0da98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"11a725162d782d59307e54b01812f83276c8b768291c214985b0e7cf30d62295","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0014f01f8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gxsms.apps.aro.cilium-cl.dvbe.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0014f01f8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gxsms.apps.aro.cilium-cl.dvbe.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"823ddd4111c05b0f17ef29326ce2923a121447ca620b68d77b77c61617a45cab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"8c6b27faba09bc535bc427b19d9416db95aeff2cd1af3b110ae29985fbe189dc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-n56r6d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0acc0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2cnps.apps.aro.d2b3z1z0m9t9r0w.e21l.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-n56r6d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0acc0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2cnps.apps.aro.d2b3z1z0m9t9r0w.e21l.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"9ea00cfe5a8003090f7394473410bf516b93ae646eaf41d11fe0c38f78877a1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"4f5c940afced23891d414938cb4cb38d59030d0411748cce89554393d71b5e2c","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002b9e330\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002b9e330\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded"}},{"row_id":"a6ef833b4aa58a2be2dade677ae06a15838ed3974fcb10a8e6438ccc3efbc708","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"5aea9a96af15785f8a001a629201e85cfd8f31b12c83f95c9952b9431240bf9b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-1-6lcjm2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9f400\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lspbg.apps.aro.e6y8v0f7c1f6x7s.ajv6.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-1-6lcjm2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9f400\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lspbg.apps.aro.e6y8v0f7c1f6x7s.ajv6.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"a74b29250bb128aea2fc8a8e43d4f1108f6a13caf904a5a84ff544ae594b4ebf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"749b338f0f666d9846dc55dd5356b693a9e5f5c64374dd55730a6f912a7a0323","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002bd0580\u003e: \n node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.005s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002bd0580\u003e: \n node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1"}},{"row_id":"b6072d80e3ae1ec169e0347b1ce3f2470c343708bd76a64cda8931e865d8852a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"239e965421538be1363dd2c7f07fdf5811ab39dbc83ef7b4d067112bb5d9f0fd","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd00d8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-c7f6w.apps.aro.cilium-cluster.dfkp.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc000fd00d8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-c7f6w.apps.aro.cilium-cluster.dfkp.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"ba06e88c84332f71f5bc3b29dbadc5823d41e75a6c0fa304def85a28900d6058","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"c5fb4bff03f86320f1edc3eea596547758a86968f42b91dffea826dc780ab8c0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-22-lzz9cz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e1c0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-m7478.apps.aro.h8z3n4k9y6j7n4d.4zzc.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-22-lzz9cz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102e1c0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-m7478.apps.aro.h8z3n4k9y6j7n4d.4zzc.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"bd8ba59464d3dcd3c86a54073647891b78add1fbcc9a911cf19e9b4978fe9c27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"3cd272fe93bf03fdbe630ac01847905ab2758c75271e802033f6c152ac24510d","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00161a240\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jnmvg.apps.aro.agg-cluster.m5gg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00161a240\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jnmvg.apps.aro.agg-cluster.m5gg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"d07ecafa5bccf9fcb34999f3963d379c068fae31c5a099cbe81347942a54170c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"fdf84045828cfe2df5b8ebc4b5d8480c96306c6e7327c688db9c24bc34502f64","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-tsnl4j\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000718740\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-rkd6v.apps.aro.s9z9e1v4q0q2t9c.1elx.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-tsnl4j\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000718740\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-rkd6v.apps.aro.s9z9e1v4q0q2t9c.1elx.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"d100558c4e197e04a283a07264a27341523eae1e39ba994c63358288c3ac4b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"6183ddd6caf2d7afde8bcd7be49b7fe37da55024a657dec1d775306e9477f62a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00112de30\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-21-184859, targetMinor=5.1.0-0.nightly-multi-2026-08-21-161733) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-21-161733\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00112de30\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-21-184859, targetMinor=5.1.0-0.nightly-multi-2026-08-21-161733) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-21-161733\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"fe1f0a20e3d9571541709af7e04a8b7c738a89bb9162bc9ed645f397a1d20587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority"}},{"row_id":"ff075b57364bb29f5cccda256427b71ce869f5905236d6b0726f0b492b2f8c06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","failed_at":"e2e","signature_id":"b7e9a22c6cf3ac389f2a77fd9b2625cddcf6fd44e337c61d636e8df2c2743c61","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ba5d28\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-21-221014) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-21-221014\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ba5d28\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-21-221014) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-21-221014\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-22T02:00:44Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"018c33755d6d533a4fdef28cd97ff4c633e9bbaae967514d687d0c45ebe7c40f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"417bf2d786aadfe3f04e5550d280ceae37c25a8bf717f073a992b1ac95f1cad4","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017e7ce0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0017e7ce0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"1db0e9021e61ee11b33548ba5f3881aa4480986f00611fb1a6b42d2bf5475b42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"7d8117c22146bf5a0774f739c19b87fd0cbd57d6a02884d31d96768d87ca983d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-cbng9k\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061a580\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jphkj.apps.aro.x5q5b2s9l3z0a5u.918u.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-cbng9k\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061a580\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jphkj.apps.aro.x5q5b2s9l3z0a5u.918u.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"295f7dbe01da480881392d4c1af1e4eb6f56ae6e3d283a37c495adabca87d9fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority"}},{"row_id":"2cae387399adad612d8489fe36e3803ecd530ed05990966b2f799f38513dc642","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"9a9e1cb7ce25683e8a9ba03d52e031fa731f92999b78652a943b0a33011dc620","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-czgrlb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e1c040\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-bgrc7.apps.aro.v1k8j6l6v9l9u6a.m0gh.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-czgrlb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e1c040\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-bgrc7.apps.aro.v1k8j6l6v9l9u6a.m0gh.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"3db7763e45b98760503c8996b6099764c9ee58ff2e4a61d4a9ce721edc3bc846","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"1b080e7bcd141068d87382a1bcd086ccda9b6dd57cad2be71077c408d2e7e438","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001588c18\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-qk46n.apps.aro.agg-cluster.s9td.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001588c18\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-qk46n.apps.aro.agg-cluster.s9td.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"619340f2588383825a4a80bab412d16d826fdfa8dc7ec33069c63937a193a389","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"f0207e912b5808bacb31a7fe73aaf94e3c55660884e013054934fb98f7a335f8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-5rl7cj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e627c0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7t6rq.apps.aro.v7b0n5l6h4h1z2z.23pj.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-5rl7cj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e627c0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7t6rq.apps.aro.v7b0n5l6h4h1z2z.23pj.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"682300c84d0ea9cbd55bc5d2d0b1043963b99042dbad5e4a4d364eebe264ab74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"15cc422b2f3be8fc9e52950819b6911fd45c6c141421a6134c4d451278306bca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-crvtsc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c4e60\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-kjwl5.apps.aro.g6t0c8c5y2f5g1s.abrk.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-crvtsc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c4e60\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-kjwl5.apps.aro.g6t0c8c5y2f5g1s.abrk.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"7a1e4f6a6a8114957971ad31b9c7ef69cac3785ccf1f6d7ee6618915b5a9ce09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"8a15fba718b6d75d8d44dad039889bf9f5ba45b37ef0458c98c5fa51b3f5ce6d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc001716000?, {0x7f5ef448fe78?, 0xc000f7dd10?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1039 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000aceb00, {0x7f5ef448fe78, 0xc000f7dd10}, {0xc001436078, 0x17}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:464 +0x2a9\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f5ef448fe78, 0xc000f7dd10}, {0xeb544f6, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:88 +0x6a5\nreflect.Value.call({0xc477720?, 0xf1650a0?, 0x13?}, {0xeb54646, 0x4}, {0xc000f7dda0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcca\nreflect.Value.Call({0xc477720?, 0xf1650a0?, 0xfc03900?}, {0xc000f7dda0?, 0x20?, 0xc755ac0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0xc477720, 0xf1650a0}, {0xc000f5d6e0, 0x2, 0xc755ac0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0xfb6b1a0?, 0xc000f7dd10?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXX Hosted Clusters - Stage - 00' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc001716000?, {0x7f5ef448fe78?, 0xc000f7dd10?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1039 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000aceb00, {0x7f5ef448fe78, 0xc000f7dd10}, {0xc001436078, 0x17}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:464 +0x2a9\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f5ef448fe78, 0xc000f7dd10}, {0xeb544f6, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:88 +0x6a5\nreflect.Value.call({0xc477720?, 0xf1650a0?, 0x13?}, {0xeb54646, 0x4}, {0xc000f7dda0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcca\nreflect.Value.Call({0xc477720?, 0xf1650a0?, 0xfc03900?}, {0xc000f7dda0?, 0x20?, 0xc755ac0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0xc477720, 0xf1650a0}, {0xc000f5d6e0, 0x2, 0xc755ac0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0xfb6b1a0?, 0xc000f7dd10?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXX Hosted Clusters - Stage - 00' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b5ec0cbb59802a88f2bc42deba73fa247a353c118d64efe8e80ca88b66f3f5b","failure_pattern":"Test Panicked","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked"}},{"row_id":"a40fd1e485f40b1d68ae27871be0b9b23f673124a0e740f1013d17c7caa7b2c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"aa7e5f753e6e6e31752d40a0be892d0d8394c3a4ed7f539a58571438102dcd83","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886800\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7qtvldntpktk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\",\n \"name\": \"ad306216-5622-4121-8bf1-f015261fa044\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-22T02:03:58.055302295Z\",\n \"endTime\": \"2026-08-22T02:23:05.756072788Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886800\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7qtvldntpktk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\",\n \"name\": \"ad306216-5622-4121-8bf1-f015261fa044\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-22T02:03:58.055302295Z\",\n \"endTime\": \"2026-08-22T02:23:05.756072788Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad1512727375aaad3a9a56a142d3171fc520281ab6edc82814158c1f09dc4aa7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"907dec55a9074061a33272193d894502c57d11b65a9f5f49e62962d638dd0ba1","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00159c228\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-cr4v4.apps.aro.cilium-cl.85qg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00159c228\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-cr4v4.apps.aro.cilium-cl.85qg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"c75f87c0eb54a53963508bcf44c7d0236900e97047dcc780c827e96b0e3db382","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"da606e988e3b163984f1144b1c2c0f76f91f7a639869fb04b4b28da971b69c0d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000517020\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9zbjjzhbwsc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n \"name\": \"8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-22T02:03:58.196286523Z\",\n \"endTime\": \"2026-08-22T02:23:05.606457627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000517020\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9zbjjzhbwsc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n \"name\": \"8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-22T02:03:58.196286523Z\",\n \"endTime\": \"2026-08-22T02:23:05.606457627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c79eeaddc922af216a19ecf8937e03e0e79f0f18f8fa145b7f805583f1ba0cea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","failed_at":"e2e","signature_id":"db6298b2d5a06ac0c0b6891a012777daa148bd9f9827d3298b5f01d7948507b1","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0013861f8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jjfp6.apps.aro.g7x2y2g3c6g1s3t.27yq.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0013861f8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jjfp6.apps.aro.g7x2y2g3c6g1s3t.27yq.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["0b5ec0cbb59802a88f2bc42deba73fa247a353c118d64efe8e80ca88b66f3f5b","3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}