{"meta":{"date":"2026-08-20","timezone":"UTC","generated_at":"2026-08-25T21:19:19Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-20T23:00:08Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"10ef907eed5985d1343a9703e3a0cc4c148a2d31c1a5dac823d2867e737406cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"bef58880af3f67335fb38c9962fdc80b164c51531808fd3a944b58b5abc7835e","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded"}},{"row_id":"250d3bd4a5f97d1c97d722810ad82884dda6229e4f6aeec2a33f3c5b47a28bc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"846c6382df9acef0fa5ddc2a6d1f4dc3f653f6d1f1033da3dc950698cda0412f","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8e13d55daacf4ddf74bdc040314e449b308767e96d40780e45800466f812928","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e","search_query":"list nodes (nodePool=npdg-422-420): Get \"https://api"}},{"row_id":"3076a8a3d12c8efd824af2c7bfb29c5e2249cfefb1c68ef893e960db7f285eb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"28d3b5e4d6e1e3b6c0885d78d546ff6d719fb6e7e972de708efbba62c246ae05","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bd4fdaffaf3b15f286ad5f2ca30cb95f7541a63bca5896bf32cc0ecd46a82996","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"\u003curl\u003e","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api"}},{"row_id":"30f62d4abaf721772d71844783bd5faa1393860acb39493884d427b31c10c1ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"4649980ccb572a8df4fef4e44831f1de4eae2d176d8ecffa92fb99f1ae87aa7d","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0025d2270\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7lgrn.apps.aro.agg-cluster.zfzg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0025d2270\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7lgrn.apps.aro.agg-cluster.zfzg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"39e16f726e18a8018f2634f811bf29bd9b6aa11618d5e6d03dc96ffbdeb07af1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"ee48f986fd9e592059d2a5cd09e88c38d974e8948f1e0f0065c5a8641bfef3cd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f170a7927803930d63ef797276984cf62fcbd2f07de426245306d08da19e2bb9","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api"}},{"row_id":"929bef3cf88497148bb10adc3e13889108acaafed0547e58e2bfbc3a71b2133d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"fdab187cfe558b4bb3f41aa7470064acf21835a87bb5df9e4d6a394f761d1b45","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded"}},{"row_id":"aa80b416e116bf10464516a724d38a5202a74a8de324a1624a363e7e86a1e244","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"3519eaf3902036b2df340ad7c69439459fca96026f79aacac0e4ada2e8cb9ab4","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8e13d55daacf4ddf74bdc040314e449b308767e96d40780e45800466f812928","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e","search_query":"list nodes (nodePool=npdg-422-420): Get \"https://api"}},{"row_id":"b032e991309f02a7d8cdef45f95a732c613d8a352824fdae4eeac35df35d9144","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"ee2ed8274c95a8959d110687ae9ece838bb447f44f7c22e5c06d8ad3a3a7f27b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f170a7927803930d63ef797276984cf62fcbd2f07de426245306d08da19e2bb9","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api"}},{"row_id":"cc96d84e9025af229f055bc2cc4105290a441d86bf5dd04c8733544e81461d8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"5b48ea80fc398a034dba3d46b19e592386ab5e5708b0a0e23267c0282b5a55be","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027ec2a0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-cxh78blbgmpq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027ec2a0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-cxh78blbgmpq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded"}},{"row_id":"fc4d8908c794302762b751fa603dac546bc5145d379bf93f60b111e5a8348053","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","failed_at":"e2e","signature_id":"9597e11d65e1b6b9e701068b23031343892ee3c2b207fcc034b59895dd98d0c1","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b748a3b1c1b9c7fd8f7f617ddf190f4c65999a469c2f67349a1718ac3211e788","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","b748a3b1c1b9c7fd8f7f617ddf190f4c65999a469c2f67349a1718ac3211e788","bd4fdaffaf3b15f286ad5f2ca30cb95f7541a63bca5896bf32cc0ecd46a82996","d8e13d55daacf4ddf74bdc040314e449b308767e96d40780e45800466f812928","f170a7927803930d63ef797276984cf62fcbd2f07de426245306d08da19e2bb9"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090538506672672768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-20T20:36:34Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2d82feb8f66a0fdd38249589cefc7a99b9bbb7caae3249786373d95c224e42f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090538506672672768","occurred_at":"2026-08-20T20:36:34Z","failed_at":"e2e","signature_id":"75e8134db7c38c2272b0135b9ef67b8e4ba756be9b98292cf44f0135c9355207","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-20T12:36:44Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"19987cab35b15d3a61ee512cd4219e7f77b20268fe8c346f213c9ff92f48d6f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","failed_at":"e2e","signature_id":"fd1a222e88fac51899884139c985b18a14bda354bbdc6d03b6bd0f1ec3b9916a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe3060\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8vkz6566fxqz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"name\": \"140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:33.753602969Z\",\n      \"endTime\": \"2026-08-20T13:35:40.453844587Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-autoscaler, cluster-node-tuning-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, azure-cloud-controller-manager, cluster-network-operator, packageserver, olm-collect-profiles, redhat-marketplace-catalog, openshift-apiserver, dns-operator, catalog-operator, olm-operator, cluster-policy-controller, cluster-storage-operator, machine-approver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent, kube-storage-version-migrator, kube-scheduler, openshift-route-controller-manager, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe3060\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8vkz6566fxqz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"name\": \"140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:33.753602969Z\",\n      \"endTime\": \"2026-08-20T13:35:40.453844587Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-autoscaler, cluster-node-tuning-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, azure-cloud-controller-manager, cluster-network-operator, packageserver, olm-collect-profiles, redhat-marketplace-catalog, openshift-apiserver, dns-operator, catalog-operator, olm-operator, cluster-policy-controller, cluster-storage-operator, machine-approver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent, kube-storage-version-migrator, kube-scheduler, openshift-route-controller-manager, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1d7d867eee3882326dc77323849dfc7e438c6412548d66950fd90c682f68c58e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","failed_at":"e2e","signature_id":"382e53c6f5440687f3efd2f0f735d848f4f2a75c27dbddc1aa370d48842e5ac7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"52d8d7e98dae1aad59b979b852812778d86130b607570c4947bc5440702dd577","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","failed_at":"e2e","signature_id":"c98eeb6d2064c80a5c88dac50a9f84605cf8ee1eb70a70345435f7284fcc94c7","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d030d37ff25b822d1e7a674fc80d6e1e263a6697d84cd3b348a474d44db8cb89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","failed_at":"e2e","signature_id":"bf90b8b00390291912ae577c4eaa041b4552832f8c625cf2cd2cff628c244b5a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000de51e8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2lhgp.apps.aro.agg-cluster.v7n3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000de51e8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2lhgp.apps.aro.agg-cluster.v7n3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"e36b3b22f808dc6d9312680c70c1b323d72cd1f7d9302e145e2351ad29d57235","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","failed_at":"e2e","signature_id":"fd7348293825c9d41446d338b7d31ffd23035dfeb87dc5ea60ab54b00dd1a82a","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc4980\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2qbp55nw5hq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"name\": \"3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:17:04.908245766Z\",\n      \"endTime\": \"2026-08-20T13:36:10.801854971Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, konnectivity-agent, catalog-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-controller-manager, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, olm-operator, kube-scheduler, dns-operator, ingress-operator, csi-snapshot-controller-operator, ignition-server-proxy, certified-operators-catalog, cluster-storage-operator, ignition-server, control-plane-pki-operator, olm-collect-profiles, openshift-route-controller-manager, cluster-network-operator, cluster-image-registry-operator, machine-approver, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-autoscaler, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc4980\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2qbp55nw5hq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"name\": \"3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:17:04.908245766Z\",\n      \"endTime\": \"2026-08-20T13:36:10.801854971Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, konnectivity-agent, catalog-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-controller-manager, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, olm-operator, kube-scheduler, dns-operator, ingress-operator, csi-snapshot-controller-operator, ignition-server-proxy, certified-operators-catalog, cluster-storage-operator, ignition-server, control-plane-pki-operator, olm-collect-profiles, openshift-route-controller-manager, cluster-network-operator, cluster-image-registry-operator, machine-approver, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-autoscaler, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e98757e3346c8fc5bb64ca20361d2a6528cc4e0fc7f5b296fd04adf5036aaa88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","failed_at":"e2e","signature_id":"37da2bd93766d6d4d4f6961c9e8500becdfb67c804e717112af85f0368264347","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6722502d5d665731952f221ae4eff4c1fe60411f08b65ee3c8c1ea7fe37c4a2f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","6722502d5d665731952f221ae4eff4c1fe60411f08b65ee3c8c1ea7fe37c4a2f","7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-20T08:35:04Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"2ed156668d2f0ed8b1300471b6b28d5fb74a014700ba29f16747d16c9389f6d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"3f5ac4cdd7ff19e9ff918b763fc044b6a568c83bba923b1245e9a1c9566d493d","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3ef7916ed999aacd2d4505cfd4c3eaabd6d5da9896782079814ea2ad7320409c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"56f9bbbf6c1cff5e3a43f7b42723ff5d2896274197334b06141ae6c0de7f3f12","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b026c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-447nnn7lv7ss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"name\": \"68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:54:41.378616139Z\",\n      \"endTime\": \"2026-08-20T09:13:49.738147219Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b026c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-447nnn7lv7ss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"name\": \"68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:54:41.378616139Z\",\n      \"endTime\": \"2026-08-20T09:13:49.738147219Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"596f050e1e2ef6bf65e38665fb11fe34a38181690255460c3551f5d85839968c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"0facf4aca2cbbc09d1d4d2b5f2a2c443dc9ac7bce839a50cfc50190997e06fd4","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000895308\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list7lw9mf\" in resourcegroup=\"ea-list-2lm6qlj6qrb9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"name\": \"fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:55.575829675Z\",\n      \"endTime\": \"2026-08-20T10:09:02.513171982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7keqbo3pl38n1vug7bjfptpeikrn1 still exists (deletion dispatched at 2026-08-20T09:44:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000895308\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list7lw9mf\" in resourcegroup=\"ea-list-2lm6qlj6qrb9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"name\": \"fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:55.575829675Z\",\n      \"endTime\": \"2026-08-20T10:09:02.513171982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7keqbo3pl38n1vug7bjfptpeikrn1 still exists (deletion dispatched at 2026-08-20T09:44:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6ee4f7993094467c454e88f6e07b3eb84afe570c5727a51452be92f0b934c7e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"1fe08479e59e5ed4afa0e1f161e992bf328632b651a998809d33c2ece866d07c","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"745effcbbe55a8bfc856a25043f331a2f47e60d5bcf3266ee741f95a1e3966cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"e0f6f930a179a3343bf2fd68ac2daa5ec05546590115b913738327ca0e47fe4d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"76e65ad13f8b705d71a0ab2e4b458d0acc2f79387ecd5c8316e0163397937ae9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"3aeb591de359ede5d82e500a19257072d85b9193e57ac8f4a13b125e2c1b0f69","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"889f947afb1e283ce1857126841237c134a552af017015e95bed19728d789ffc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"50cf928a4b9ab91ad463923b6c69c34145bea77427cbfe46d31360d594882c1c","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7356cf97f840631ab4a0a63ce4f36ca842474a0ca4160d56ce50ad927de67954","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9412accf6fc59ffd6997c17f5568c15c3b71bdef31ec19ab86acc060182c51c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"f2f000bdc64541602873ebc887a909036d077a88b2e4d46c3fb4c2246d3285bd","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae0da0\u003e: \n    failed to create HCP cluster np-downgrade-wt9mk2: failed waiting for cluster=\"np-downgrade-wt9mk2\" in resourcegroup=\"rg-np-downgrade-wt9mk2-ftnm4xhg5vhq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e349a722-afad-443f-add1-19a731db69f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e349a722-afad-443f-add1-19a731db69f3\",\n      \"name\": \"e349a722-afad-443f-add1-19a731db69f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:41.289744282Z\",\n      \"endTime\": \"2026-08-20T09:55:41.524756796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae0da0\u003e: \n    failed to create HCP cluster np-downgrade-wt9mk2: failed waiting for cluster=\"np-downgrade-wt9mk2\" in resourcegroup=\"rg-np-downgrade-wt9mk2-ftnm4xhg5vhq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e349a722-afad-443f-add1-19a731db69f3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e349a722-afad-443f-add1-19a731db69f3\",\n      \"name\": \"e349a722-afad-443f-add1-19a731db69f3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:41.289744282Z\",\n      \"endTime\": \"2026-08-20T09:55:41.524756796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a005c8f75388cab6c2b3cd6f256077bc533e9eb924f5d6baa3292ecd6fb4b9be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"30e3ce44a70a32080f494a46962fb9e3504a602768cbec5e3cc893e94e06d32f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2586ba17a3b1ece85ca1e02cb96ed1d45f2550a44e267258b7c98cd39894bd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"7ebf3a7566f82ffac1f2fdc94973962a97b9cebbccee3a25837ecc58e0903fb4","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000caeac0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lfvfr4f2rlsg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"name\": \"f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:43:33.944124188Z\",\n      \"endTime\": \"2026-08-20T10:02:41.325901249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000caeac0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lfvfr4f2rlsg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"name\": \"f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:43:33.944124188Z\",\n      \"endTime\": \"2026-08-20T10:02:41.325901249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d033cbe456c40816d8a9091a7f92d3ca34b62870ff5c9210e4beb05647639461","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"84a396fab7c2ab3ae1588023933d5e474ab7b143128d353980cd15000536f48b","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b6bc70dde3370d5e666a11dcdf8b092a80443f417ead980036aeb5b45355b1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, ignition-server, redhat-marketplace-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"df6bb02b94b598111a9250e28dd8bae6ad4abe649dff41ea95699699d98c67cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"f86b3413a4fa89f0fe3aee0a97c3b3aae586cbd24a62a162177322eec5c926c9","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6ed60\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-vp6x5vhlr7vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"name\": \"0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:30:45.72149794Z\",\n      \"endTime\": \"2026-08-20T09:49:51.406315863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server-proxy, dns-operator, ingress-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, openshift-controller-manager, openshift-route-controller-manager, packageserver, olm-collect-profiles, machine-approver, olm-operator, control-plane-pki-operator, openshift-apiserver, cluster-policy-controller, hosted-cluster-config-operator, redhat-operators-catalog, community-operators-catalog, catalog-operator, ignition-server, certified-operators-catalog, konnectivity-agent, cluster-network-operator, cluster-autoscaler; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6ed60\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-vp6x5vhlr7vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"name\": \"0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:30:45.72149794Z\",\n      \"endTime\": \"2026-08-20T09:49:51.406315863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server-proxy, dns-operator, ingress-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, openshift-controller-manager, openshift-route-controller-manager, packageserver, olm-collect-profiles, machine-approver, olm-operator, control-plane-pki-operator, openshift-apiserver, cluster-policy-controller, hosted-cluster-config-operator, redhat-operators-catalog, community-operators-catalog, catalog-operator, ignition-server, certified-operators-catalog, konnectivity-agent, cluster-network-operator, cluster-autoscaler; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"df9fbe07981fc948a333e7848059ac81d6bfb10da405050c727d823d752e313a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"a1399ba06ca55ef9ac4602a99872b2b1d13798c7ec52bf26477719dd69c14e17","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001084138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9zcvtfmqbq5k\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"name\": \"d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:46:19.133487927Z\",\n      \"endTime\": \"2026-08-20T10:10:22.487349206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kgqae0lr73vi3huktro85eu2akrr still exists (deletion dispatched at 2026-08-20T09:46:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001084138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9zcvtfmqbq5k\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"name\": \"d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:46:19.133487927Z\",\n      \"endTime\": \"2026-08-20T10:10:22.487349206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kgqae0lr73vi3huktro85eu2akrr still exists (deletion dispatched at 2026-08-20T09:46:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8ad89dd910bc012411710f4ee306b2dfe8c016cdadfa6f0b4948541c9b5196c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","failed_at":"e2e","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","4b6bc70dde3370d5e666a11dcdf8b092a80443f417ead980036aeb5b45355b1f","59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","7356cf97f840631ab4a0a63ce4f36ca842474a0ca4160d56ce50ad927de67954","7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6","de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-20T03:16:28Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1d63eb2956a3569e962a255189f37aa472731f409f5965a56224210b0267c104","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","failed_at":"e2e","signature_id":"7e2cf5d1bff2be054d231178fc2524db6c79b0da8fc042c6972e38c1b9001d27","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"22ddb7050fcdf24af1a6c6fbad266a744cbd32f13d679e25195bc70eff4ad09f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","failed_at":"e2e","signature_id":"e14f3f9d9bbacbaff6e558f257e54562c56eab915ed893444a9648f95512b4c2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-20T02:00:07Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"aee7d24a8fbc1277e0bd4ae046b4722ee18ba17a31be42af28277f717a51d05b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","failed_at":"e2e","signature_id":"8e749cbda0987bd36e3427fb17c74e6efb4a7ffd7f85d804ab73f254f7c2b973","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce22653c1790596420acf5ec30dfcb3f90e95095df0cfef47397e47b57f8997","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"c0143155e16ada1240e6d031e3eb1df03a69fdba340293c851270cc458549756","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","failed_at":"e2e","signature_id":"d19ec4d4f31d760d02b50b389f562a65f23856356a3f633669a10c48565d949d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00176e318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-hkbsbzplhbxk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6deb1d14-3e14-491b-b92f-53c5344ce486\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6deb1d14-3e14-491b-b92f-53c5344ce486\",\n      \"name\": \"6deb1d14-3e14-491b-b92f-53c5344ce486\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:19:48.926061657Z\",\n      \"endTime\": \"2026-08-20T02:43:48.575119816Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa13vuihbvlq5ba5p02g5kj4iq47j36 still exists (deletion dispatched at 2026-08-20T02:19:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00176e318\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-hkbsbzplhbxk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6deb1d14-3e14-491b-b92f-53c5344ce486\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6deb1d14-3e14-491b-b92f-53c5344ce486\",\n      \"name\": \"6deb1d14-3e14-491b-b92f-53c5344ce486\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:19:48.926061657Z\",\n      \"endTime\": \"2026-08-20T02:43:48.575119816Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa13vuihbvlq5ba5p02g5kj4iq47j36 still exists (deletion dispatched at 2026-08-20T02:19:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8edac4562f39750f52a51476c224546f9aec4669096a1f2770e599e90dd95c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","failed_at":"e2e","signature_id":"2c025bacef2a95219db5cc2c1d0f8e25156096d570a86e526e43447bf8f9a29b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dgkn5p9l9flc/cluster-candidate-5-0-nrmbvd should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a85590\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dgkn5p9l9flc/cluster-candidate-5-0-nrmbvd should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a85590\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded"}},{"row_id":"f147525af9f86d6a2698524fb7dcdb1489205b40d6aadcc56a41cc65259d241b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","failed_at":"e2e","signature_id":"4ebee03d5cb8560c61e868d8bbb831f73868f90ba39c7b26a29c917e8892dc28","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8dad9a32124d40126133bba2520cede07e7f2a6cd0df33b90c1e690aeb9f3131","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"ff2b476bfff5ebc4178015fc7845bf03a3c6cf7ca57f61c28b359f34de69b5ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","failed_at":"e2e","signature_id":"2e88e71b6a16994e8d9342a6dfffd6392efeb37bdd6569b14205b6f91d0d4a24","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4a4f87d4133fd3a9df163a91b47668459c543343ba1d123ba27a73ceaa3385ce","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["4a4f87d4133fd3a9df163a91b47668459c543343ba1d123ba27a73ceaa3385ce","59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7ce22653c1790596420acf5ec30dfcb3f90e95095df0cfef47397e47b57f8997","8dad9a32124d40126133bba2520cede07e7f2a6cd0df33b90c1e690aeb9f3131"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}