{"meta":{"date":"2026-08-21","timezone":"UTC","generated_at":"2026-08-25T21:19:09Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-21T23:00:43Z"},"failed_at":["e2e"],"failed_test_count":20,"occurrences":[{"row_id":"19eae76ad76611ba561734dc48862c3df962e15122c12fa6bec6216fa66f73ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"a610245db2dc06bccba732a90d5f0ee6d5e2651fd4bb3d357ae0d19e0833cec8","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc002a08360\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-94kgn.apps.aro.z7t1m4x0s6z3p9j.lpin.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc002a08360\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-94kgn.apps.aro.z7t1m4x0s6z3p9j.lpin.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"37482449a22a9ee2838dce4d0a746faccf50fc628b8aa7b66fef3b15d4029df5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"69ff046ff55d8bfc7023c374da3602741135d47054bcf695f149cb3279b1d92b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-g9dhkm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012e2980\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gztjc.apps.aro.m2v4p9z2h5c4v6q.qaf0.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-g9dhkm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012e2980\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gztjc.apps.aro.m2v4p9z2h5c4v6q.qaf0.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"3db0e701b89943e981b0485bb22cb2bfbac24222f9f7ff65d3ce2720ddad5d8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"fbf7d38b96f28f05b21778aab3674d4cce11c0c6f81909ef1d5dc73f06ad06e2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3f2e0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sh428kn5d89n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n \"name\": \"b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:04:23.850321515Z\",\n \"endTime\": \"2026-08-21T23:23:28.45210372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3f2e0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sh428kn5d89n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n \"name\": \"b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:04:23.850321515Z\",\n \"endTime\": \"2026-08-21T23:23:28.45210372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"41489c658330c7df0f42d783c84bca0dc50982e2e5f90c5eb7a1d5fc1fb0dbfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"62bbc0ad65ca651617994b83d05a4a4fe629d762c5e177de75e71c5449cc78fa","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00158d2f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-21-140538, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-20-065923\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00158d2f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-21-140538, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-20-065923\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"5dd4d24e4382e34e2e24fc41ef0a794fcf0f668c032a52120356ef6b4b151767","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"e6a8ea22f064439192de7c94bd0b019d27fcf10b53a96a3dbbeaabdae0a1680a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0014379c8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-21-184859, targetMinor=5.1.0-0.nightly-multi-2026-08-21-161733) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-21-161733\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0014379c8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-21-184859, targetMinor=5.1.0-0.nightly-multi-2026-08-21-161733) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-21-161733\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"6104d0d9bbc232f468f3b25f45f3f65aa1b383dc8af75084b51946e589627a5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"86e1eb29ec597937a663760329209bb50694b5af2a87b7c19ee3c634d1801423","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001782858\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-215447, targetMinor=4.23.0-0.nightly-multi-2026-08-21-150158) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-21-150158\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001782858\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-215447, targetMinor=4.23.0-0.nightly-multi-2026-08-21-150158) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-21-150158\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"624cf7b9c8d653c1d34c6b9dc834b4dca8b6214b402672fb88aa767cc1c01539","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"697a4c129533a315c9016b84da122b5bc17c7516225d6ed78343e011c45dd4ea","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-22-4ntvj5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068120\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-ppc2q.apps.aro.g6q5a8i3e4o4h4a.b1f9.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-22-4ntvj5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068120\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-ppc2q.apps.aro.g6q5a8i3e4o4h4a.b1f9.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"758bd35c76738506ecfca0f6b20dc7bba5319bdb70952d73f31bc6f10a0e62cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"115305b069b0aa8d64167995814d5cd6d32a1f792870e701a3affd4a8b96bdfe","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001206090\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4t9jh.apps.aro.agg-cluster.c93g.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001206090\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-4t9jh.apps.aro.agg-cluster.c93g.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"799c043a7f7778f114e69340cafa928c44ca60710aa3c26307f2d336828a13e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"ca75b2343ff23a11e0fce1a31c9ef24459c4c103d790af5c0d8320fc327b3727","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0011964b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-h8fb74\" in resourcegroup=\"rg-np-version-upgrade-h8fb74-s5mqnw56gx4w\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\",\n \"name\": \"701376c3-ab04-47a8-a1ef-8375724f8f86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-22T00:26:59.095965518Z\",\n \"endTime\": \"2026-08-22T00:51:01.463363445Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb99fkh6uh3oo5nr4qni7jhed5o0tgt still exists (deletion dispatched at 2026-08-22T00:26:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0011964b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-h8fb74\" in resourcegroup=\"rg-np-version-upgrade-h8fb74-s5mqnw56gx4w\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\",\n \"name\": \"701376c3-ab04-47a8-a1ef-8375724f8f86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-22T00:26:59.095965518Z\",\n \"endTime\": \"2026-08-22T00:51:01.463363445Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb99fkh6uh3oo5nr4qni7jhed5o0tgt still exists (deletion dispatched at 2026-08-22T00:26:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c4ab96afc533dd46ebb4f2b4e1e6d7518f9cf6c5331351d08cb6a08673f437f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"587e72c577de17291ef8b6b3ac4f6b12a0663bf2c56e0f53f30373aec8471091","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-21-9gpzfp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880020\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-qghdq.apps.aro.f4t5y3d9c0q1k1u.vdle.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-21-9gpzfp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880020\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-qghdq.apps.aro.f4t5y3d9c0q1k1u.vdle.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"995b327c25497ccd1fab60952ded4edc3388c62cc66ae5fe0a845246e67b297d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"bf0b84a697d66c570dffb08e752d5c50734534e54f512951dc11b3757ac8cd8e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0020042b8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-n78pt.apps.aro.cilium-cluster.g8y8.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0020042b8\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-n78pt.apps.aro.cilium-cluster.g8y8.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"9fe0475ac5638995275090dcd12bc238ff8b548b92b99e068a08b2b839892b94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"014faee8c65110da8bf653c059bda71ae8ef3480a383a2999fc04af379294b99","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc000fa8228\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-bh6cc.apps.aro.cilium-cl.zseo.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc000fa8228\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-bh6cc.apps.aro.cilium-cl.zseo.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"a26db36f5d6a348caa9a7a8b43b6613382b300bcc7ffd1ed5a3a2bc169f54561","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"a1166de0889c8bcb931d1a2474d9ac58b2df20920fe3d4ae0d4b27104be80122","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-1-zkq687\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e02f00\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7hg2s.apps.aro.z9r1f7q2r7l8b0d.3h01.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-5-1-zkq687\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e02f00\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7hg2s.apps.aro.z9r1f7q2r7l8b0d.3h01.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"a4ddd6ff170ea1dc0d701fb0047dd27952a3d017f56ff81ef449b5fe4dce181b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"9f5e26191b5a76b3738feae66a80838e9ebca5f7e61655121cd70c58e39a58ff","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00291d480\u003e: \n node pool upgrade verification failed: k3s3o2p6f4s4v6d-nps-420-422-v844j-56sx7 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); k3s3o2p6f4s4v6d-nps-420-422-v844j-vrq4d (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"k3s3o2p6f4s4v6d-nps-420-422-v844j-56sx7\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9463da4154ce4a2354a72c8ac8d2b373ea30e73f8881235d77e00f3c4642f31e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:df3ebe9aea9dcc90e0e472cafb478cb56fcca54b6aa51326c3af1afdca3ab557\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"k3s3o2p6f4s4v6d-nps-420-422-v844j-vrq4d\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00291d480\u003e: \n node pool upgrade verification failed: k3s3o2p6f4s4v6d-nps-420-422-v844j-56sx7 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); k3s3o2p6f4s4v6d-nps-420-422-v844j-vrq4d (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"k3s3o2p6f4s4v6d-nps-420-422-v844j-56sx7\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9463da4154ce4a2354a72c8ac8d2b373ea30e73f8881235d77e00f3c4642f31e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:df3ebe9aea9dcc90e0e472cafb478cb56fcca54b6aa51326c3af1afdca3ab557\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"k3s3o2p6f4s4v6d-nps-420-422-v844j-vrq4d\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:119e08a093313013cf5d2a0a0591fb9caea050d90b45ea0fe2cec4626debd0c5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:99e32a4561fee20b636bd10bed42ba33d44f4a197bc729433aa2f642b19b9208\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:60c9ecc3a425aabdb9e30ab2648c40d02c303028686c1754030d2e41ab40f2af\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:662b24d01d927a57f76c9a549ae264bed965c83a4ba7bcc62520a93744319719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1a7474953c660d19e776c3aeb446693fc13fe7b498393e2b7c42428132502165\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c8c15ea20b5d920afc7aa49cbe2c134a00cc1b6f4dfb7066d5c84e6ba7bc2f0\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:49754b811af4de663301e8103b5ab276c58b98aa49f3060e185ee4c92fc08110\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e369471f4be292006624626c74fb6c7cf055218a1a3e2b5f43891a54f0e5179\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2d974b7b6f9b1b95e40b48718fa32658af382f27912ccd21a7588d1c2a072bfb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f269d55644258108fc29c0c44b433d5998e27fa03c3229d852762622012e3bf0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:68bc07fbba4df68e7243b1af9d1874df28b9fb534db84eb1854d591433d027b0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc4bc3f1504fd8cb81c13acc937bf7c8075287534e6422f2a9d56a5f4f243853\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: k3s3o2p6f4s4v6d-nps-420-422-v844j-56sx7 (no version in containerRuntimeVersion \"cri-o://1"}},{"row_id":"be79137be0bdd138617229c7517bf59c8180dfa25dda4e09ffc9667822853cad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"b880458c73aad38c09b6646920150ac274a35d4170590ae9673504a08162d18b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ae11a0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ae11a0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"d4c8d771e9379d6374ca296db8b9386bae7acf31f30dc6da9fc64bc7b467d93a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"463a01fef05dd95fc4804e0b40a98b4138b6a654495626a1d5814204f1feb3ff","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:139]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006ae100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-s82wq8qv974g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n \"name\": \"389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:17:03.253121618Z\",\n \"endTime\": \"2026-08-21T23:36:09.200090899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:139]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006ae100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-s82wq8qv974g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n \"name\": \"389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T23:17:03.253121618Z\",\n \"endTime\": \"2026-08-21T23:36:09.200090899Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd843b2d0f25d4f46b94defdbbbecf2dc93608399b5fa99924d9e620cdd1b3af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"6073bf034d1aabb50f60a9559998d196f90b404e7a6f2c7c9d69ac4bbe1020b6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-20-jjv5xb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009005c0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-h66n8.apps.aro.k8a8c6b7o7d9m1j.nkke.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-20-jjv5xb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009005c0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-h66n8.apps.aro.k8a8c6b7o7d9m1j.nkke.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"e70832c252a2f4ca444ad7e6978498e8472f0e005bfc866af8da71538f594646","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"0e428625a39042ad70067f1e3a5ffb3ef2deec8808b6df01f291fa0854988be4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-jnws78\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b809a0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gbwn9.apps.aro.t5q8q6u6n3l6q0m.xm4f.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-jnws78\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b809a0\u003e: \n [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gbwn9.apps.aro.t5q8q6u6n3l6q0m.xm4f.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"e958405d37d2bbe294feaffd4ef49e10c4cfaaed2c1edca53fe0c1352c14d0e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"9d8ff92f48ab455bb4637f037f6fb65f0f9eab5e788e3363f3d8f0a842a4a77d","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00224a8d0\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-8c8wbl887ct4, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8c8wbl887ct4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-8c8wbl887ct4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00224a8d0\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-8c8wbl887ct4, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8c8wbl887ct4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-8c8wbl887ct4, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-8c8wbl887ct4, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8c8wbl887ct4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-8c8wbl887ct4, error: context deadline exceeded"}},{"row_id":"f0ae107189e00024c98119d1af0dfc70f636dd1db99b9e3bbade2e5880dca8d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","failed_at":"e2e","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003cx509.UnknownAuthorityError\u003e: \n x509: certificate signed by unknown authority\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-21T02:00:08Z"},"failed_at":["e2e"],"failed_test_count":43,"occurrences":[{"row_id":"0295ba114b45cbce7d3a867988ab7a9c9cd1472bfc767736a39ab9cd47fea2d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"735a2a35f6b1c5e1b700cd4fe89f07201e4b403b4fd45e0a13a0acfcf9fe12f2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-gmgwb4\" with version 4.22.9 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001304500\u003e: \n failed to create HCP cluster cluster-zstream-4-22-gmgwb4: failed waiting for cluster=\"cluster-zstream-4-22-gmgwb4\" in resourcegroup=\"rg-zstream-upgrade-4-22-kprpsfjxj6br\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c8bba58d-0065-4698-a45d-4df01b5e0a2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c8bba58d-0065-4698-a45d-4df01b5e0a2b\",\n \"name\": \"c8bba58d-0065-4698-a45d-4df01b5e0a2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:38.09193876Z\",\n \"endTime\": \"2026-08-21T02:51:43.682658813Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-gmgwb4\" with version 4.22.9 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001304500\u003e: \n failed to create HCP cluster cluster-zstream-4-22-gmgwb4: failed waiting for cluster=\"cluster-zstream-4-22-gmgwb4\" in resourcegroup=\"rg-zstream-upgrade-4-22-kprpsfjxj6br\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c8bba58d-0065-4698-a45d-4df01b5e0a2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c8bba58d-0065-4698-a45d-4df01b5e0a2b\",\n \"name\": \"c8bba58d-0065-4698-a45d-4df01b5e0a2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:38.09193876Z\",\n \"endTime\": \"2026-08-21T02:51:43.682658813Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"04a2615c26a11b259b888b58d9f6d8a7caab52ac1cac8dd781c86fd3ed45b8d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"ae9917932f90b2d8b738c6ed2ec73fd085b17fa53168e8719072bde1b10a9a0f","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-dn2shm at version 4.22.10\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009cce40\u003e: \n failed to create HCP cluster np-skip-minor-dn2shm: failed waiting for cluster=\"np-skip-minor-dn2shm\" in resourcegroup=\"rg-np-skip-minor-dn2shm-w2tw928ph8rh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/382c39da-c8a7-48a0-a8f4-4bbc23bd14e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/382c39da-c8a7-48a0-a8f4-4bbc23bd14e3\",\n \"name\": \"382c39da-c8a7-48a0-a8f4-4bbc23bd14e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:47:52.350358779Z\",\n \"endTime\": \"2026-08-21T03:06:54.994794077Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-dn2shm at version 4.22.10\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009cce40\u003e: \n failed to create HCP cluster np-skip-minor-dn2shm: failed waiting for cluster=\"np-skip-minor-dn2shm\" in resourcegroup=\"rg-np-skip-minor-dn2shm-w2tw928ph8rh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/382c39da-c8a7-48a0-a8f4-4bbc23bd14e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/382c39da-c8a7-48a0-a8f4-4bbc23bd14e3\",\n \"name\": \"382c39da-c8a7-48a0-a8f4-4bbc23bd14e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:47:52.350358779Z\",\n \"endTime\": \"2026-08-21T03:06:54.994794077Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0535421f2217e9ad7a4eb7ea5976689403cfa373f4fc0c17a45312eba045e8a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"afbd9ea61b7f5f2cf2d4e5eb07c2fecfd15d1778af4a847a49ff12ce110071ab","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0013c8138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2p9lgg975wsf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n \"name\": \"d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:51:19.851300811Z\",\n \"endTime\": \"2026-08-21T03:15:24.947768689Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2samnf949m9mks8utf8hvjl3i9ocmvd0 still exists (deletion dispatched at 2026-08-21T02:51:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc0013c8138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2p9lgg975wsf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n \"name\": \"d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:51:19.851300811Z\",\n \"endTime\": \"2026-08-21T03:15:24.947768689Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2samnf949m9mks8utf8hvjl3i9ocmvd0 still exists (deletion dispatched at 2026-08-21T02:51:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"12e54c00c385f1eb02a44539c5678b04d71f18d9e693a114042d8068a9e5ba7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"eaae9ffa124a963d7a6edae9a93b0de32ff8eeb2a17d21c5b6b88689e4aad9b4","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef23a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-msfxz7tckkqf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a5e8cd9-4aae-478c-9392-84b32f68c3cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a5e8cd9-4aae-478c-9392-84b32f68c3cd\",\n \"name\": \"3a5e8cd9-4aae-478c-9392-84b32f68c3cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:40:43.288756824Z\",\n \"endTime\": \"2026-08-21T02:59:43.456227188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, redhat-marketplace-catalog, machine-approver, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, olm-collect-profiles, community-operators-catalog, kube-controller-manager, openshift-controller-manager, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, cluster-image-registry-operator, konnectivity-agent, catalog-operator, certified-operators-catalog, kube-scheduler, cluster-storage-operator, ignition-server-proxy, packageserver, control-plane-pki-operator, openshift-apiserver, hosted-cluster-config-operator, dns-operator, ingress-operator, redhat-operators-catalog, azure-cloud-controller-manager, ignition-server; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef23a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-msfxz7tckkqf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a5e8cd9-4aae-478c-9392-84b32f68c3cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a5e8cd9-4aae-478c-9392-84b32f68c3cd\",\n \"name\": \"3a5e8cd9-4aae-478c-9392-84b32f68c3cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:40:43.288756824Z\",\n \"endTime\": \"2026-08-21T02:59:43.456227188Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: olm-operator, redhat-marketplace-catalog, machine-approver, cluster-policy-controller, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, olm-collect-profiles, community-operators-catalog, kube-controller-manager, openshift-controller-manager, cluster-autoscaler, cluster-version-operator, openshift-route-controller-manager, cluster-image-registry-operator, konnectivity-agent, catalog-operator, certified-operators-catalog, kube-scheduler, cluster-storage-operator, ignition-server-proxy, packageserver, control-plane-pki-operator, openshift-apiserver, hosted-cluster-config-operator, dns-operator, ingress-operator, redhat-operators-catalog, azure-cloud-controller-manager, ignition-server; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1463425f3b21027f6cd4a021d07c248f1bdb18d000fc07ee56213c9c68098f52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"82fc9dfcd4a1d1dc958933049664f71eaf4c60ebd9aac5ce72d39c5595e18a42","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e98d20\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-x7drbm8v2rwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28269472-f9ce-447f-911b-7d8d67a613c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28269472-f9ce-447f-911b-7d8d67a613c4\",\n \"name\": \"28269472-f9ce-447f-911b-7d8d67a613c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T03:01:43.758642922Z\",\n \"endTime\": \"2026-08-21T03:20:44.77388246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e98d20\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-x7drbm8v2rwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28269472-f9ce-447f-911b-7d8d67a613c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28269472-f9ce-447f-911b-7d8d67a613c4\",\n \"name\": \"28269472-f9ce-447f-911b-7d8d67a613c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T03:01:43.758642922Z\",\n \"endTime\": \"2026-08-21T03:20:44.77388246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1b28b7178658b2a828d0cc52a2ed28a85c501e4aab206a55308e7d0a6fc02acb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"9b7bbad63103e55151b58e9fe0aa82ebcd0a55b9322b35d817145a7e77a3fb08","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2e7a0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dhdl29xp87ff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n \"name\": \"94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:39:56.502029488Z\",\n \"endTime\": \"2026-08-21T02:59:03.450785892Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2e7a0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dhdl29xp87ff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n \"name\": \"94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:39:56.502029488Z\",\n \"endTime\": \"2026-08-21T02:59:03.450785892Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bae2c31dfd388275f409339683e6035aa6871992170a42fd34eae18131b6e2ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2008b4a48ca9680f285cd477622992a7315b9d954eb8281e74a9076d4d0257b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"a653dd0fe3070ac575df986a0df9a1327f00fa60786168762feb747e19bdb7f7","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-52rvxg\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018c2cc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-52rvxg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-52rvxg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sht9lcl99cmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n \"name\": \"ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:33:22.674679438Z\",\n \"endTime\": \"2026-08-21T02:52:23.424354393Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-storage-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, csi-snapshot-controller-operator, cluster-version-operator, openshift-route-controller-manager, catalog-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-network-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-52rvxg\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018c2cc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-52rvxg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-52rvxg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sht9lcl99cmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n \"name\": \"ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:33:22.674679438Z\",\n \"endTime\": \"2026-08-21T02:52:23.424354393Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-storage-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, csi-snapshot-controller-operator, cluster-version-operator, openshift-route-controller-manager, catalog-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-network-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6cef856251578d6d4f842b0ea51927280752396355aaa78eb5a3ffb310ef3d92","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b31bbcdc685991a768f95de55f6353625592b4540110c7b9b0c28bbd4cd6572","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"90540ad1f71b81bed938e2d289c44e688fd651904bd2cb8cddef646e47d0d371","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e61c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gdl8mt5sbkqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\",\n \"name\": \"4841240d-e2f8-4813-8ee4-75401f65cda9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:05:01.793828852Z\",\n \"endTime\": \"2026-08-21T02:24:02.290641543Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e61c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gdl8mt5sbkqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\",\n \"name\": \"4841240d-e2f8-4813-8ee4-75401f65cda9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:05:01.793828852Z\",\n \"endTime\": \"2026-08-21T02:24:02.290641543Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3262cf6de30369c2ffcb6579be8ed7512f4642fde958665cbc9766e8db1c3ef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"dc0cc1f46b4c3a94844aceabaa9ad9108ada785e52f771e817b2071398fc7bb1","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ee20\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9gxwf7qrq4lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/308e246f-4883-4026-9f3b-55572bf61501\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/308e246f-4883-4026-9f3b-55572bf61501\",\n \"name\": \"308e246f-4883-4026-9f3b-55572bf61501\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:18.84549218Z\",\n \"endTime\": \"2026-08-21T02:22:22.549034229Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ee20\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9gxwf7qrq4lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/308e246f-4883-4026-9f3b-55572bf61501\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/308e246f-4883-4026-9f3b-55572bf61501\",\n \"name\": \"308e246f-4883-4026-9f3b-55572bf61501\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:18.84549218Z\",\n \"endTime\": \"2026-08-21T02:22:22.549034229Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"339833af56dd44828f6c0a3093a2d5cdb257a6345667aa1a0df047e264c81f95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"0027226a133920ef1c26470ce81c33d48a4b562da304b31b0fe5ea881d2172b5","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cc1940\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-p8kxn92zjqhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\",\n \"name\": \"5185baff-cb0e-4088-baa6-c6571cadd51f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:36.75245159Z\",\n \"endTime\": \"2026-08-21T02:22:42.448207861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cc1940\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-p8kxn92zjqhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\",\n \"name\": \"5185baff-cb0e-4088-baa6-c6571cadd51f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:36.75245159Z\",\n \"endTime\": \"2026-08-21T02:22:42.448207861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3a4086686d653880c8a268a5d3b35cf7af9ecb6c8b12897ddf335b943603c2ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"5b3802d878773f1a0822d4136f1c861b8287fa1e3f87823166eff58535d9f1f9","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dba740\u003e: \n failed to create HCP cluster ea-listfncws6: failed waiting for cluster=\"ea-listfncws6\" in resourcegroup=\"ea-list-zhgflbxgws4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n \"name\": \"04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:35:24.937663979Z\",\n \"endTime\": \"2026-08-21T02:54:33.431541939Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dba740\u003e: \n failed to create HCP cluster ea-listfncws6: failed waiting for cluster=\"ea-listfncws6\" in resourcegroup=\"ea-list-zhgflbxgws4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n \"name\": \"04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:35:24.937663979Z\",\n \"endTime\": \"2026-08-21T02:54:33.431541939Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcefb659555e6a990fedc929fc65e2b32862b237e18ebc38834dc9a0eaf288eb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42073365c48e78c1633c41fab1169adbdc9a50d5fff1a345ffc9f38b249b9981","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"0c797b790f34212a0962c3e54028dc04c84df9396c80443411635e702cab7abf","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cc9300\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6cppfxw8g4qh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n \"name\": \"1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:28.697047474Z\",\n \"endTime\": \"2026-08-21T02:22:32.526910358Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cc9300\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6cppfxw8g4qh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n \"name\": \"1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:28.697047474Z\",\n \"endTime\": \"2026-08-21T02:22:32.526910358Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4290faaf4dfa13dcff07ad479bc77608dc11fabffe30e7f45fd239442e20af9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"96a2fe2e929f73bfb9adc313d26af9eac1da35f5933f0823653985b4fdc7f529","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-z2b7swxgrmhl\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0039aa420\u003e: \n failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-z2b7swxgrmhl\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0039aa420\u003e: \n failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded"}},{"row_id":"42d917953e9738aa1fdaa4ca0a7ed35b7877c109ec4eff35ed957e058d6ca43c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"942360ff8169c2a19cebe856cddf17d577b93e9f7408dcd1cd1902c9daf1c0df","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-qjpmtqf7lz96/cluster-candidate-5-0-fb6xgr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881960\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fb6xgr: failed waiting for cluster=\"cluster-candidate-5-0-fb6xgr\" in resourcegroup=\"rg-candidate-5-0-qjpmtqf7lz96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54bfa2aa-72cd-4b76-9bdd-eb33acc0c0bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54bfa2aa-72cd-4b76-9bdd-eb33acc0c0bb\",\n \"name\": \"54bfa2aa-72cd-4b76-9bdd-eb33acc0c0bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:22.283245268Z\",\n \"endTime\": \"2026-08-21T02:51:23.609714741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-qjpmtqf7lz96/cluster-candidate-5-0-fb6xgr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881960\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fb6xgr: failed waiting for cluster=\"cluster-candidate-5-0-fb6xgr\" in resourcegroup=\"rg-candidate-5-0-qjpmtqf7lz96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54bfa2aa-72cd-4b76-9bdd-eb33acc0c0bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54bfa2aa-72cd-4b76-9bdd-eb33acc0c0bb\",\n \"name\": \"54bfa2aa-72cd-4b76-9bdd-eb33acc0c0bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:22.283245268Z\",\n \"endTime\": \"2026-08-21T02:51:23.609714741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"46c246b9a5f319e6e9a0e2243a6f5f822933efd81d9594c835e6b83801540899","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"f4ef26d460058d90ed4ee949e2b53c1556150d27ed9ab6fc4a76d0e97cfd95ed","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012312e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mm9t5htdp974\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\",\n \"name\": \"e464b7e0-c213-4e6e-988f-74f58142aa41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:05:47.205015937Z\",\n \"endTime\": \"2026-08-21T02:24:52.289709904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, packageserver, redhat-marketplace-catalog, openshift-controller-manager, ingress-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-storage-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012312e0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mm9t5htdp974\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\",\n \"name\": \"e464b7e0-c213-4e6e-988f-74f58142aa41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:05:47.205015937Z\",\n \"endTime\": \"2026-08-21T02:24:52.289709904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, packageserver, redhat-marketplace-catalog, openshift-controller-manager, ingress-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-storage-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17cbac65ad6dde679afa9f8374f5b421e45ae642715ea520e35f5231ee1e4b90","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5983d1f7009b6c3070df7d880fa408d01ec22d5b2db1bc3420e0e18fab524db6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"43f28e097ff6599e427ad726f1fa1c75940fb0b825513844f3957b76cf6b817b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f84520\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fq28q2k4mqnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n \"name\": \"7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:19.705086647Z\",\n \"endTime\": \"2026-08-21T02:22:22.572455462Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f84520\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fq28q2k4mqnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n \"name\": \"7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:19.705086647Z\",\n \"endTime\": \"2026-08-21T02:22:22.572455462Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c3dbb58a7a59f8ef86681db070676762c58ce5ff5b3bc2c30f6e31cd18cfea3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"001ed7fa69d2a092f2b072655683cfb30462642d1bd846ab3b9b253dbcf3dab5","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001a80bd0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n \"name\": \"0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:04:28.992322331Z\",\n \"endTime\": \"2026-08-21T02:23:32.540875768Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001a80bd0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n \"name\": \"0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:04:28.992322331Z\",\n \"endTime\": \"2026-08-21T02:23:32.540875768Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5fb10184e5c564488d6b61b261d39089d112f057efc413f1fa48796ca8a03d93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"01d3c79c792237fb62672cad293b186ea02a3ea2191a33420c6fdd434b27d6f3","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0028eeae0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92dczdzgqxgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\",\n \"name\": \"739a7635-b32c-4f38-8193-2975abc5e092\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:40:33.871209807Z\",\n \"endTime\": \"2026-08-21T02:59:33.678476291Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0028eeae0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92dczdzgqxgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\",\n \"name\": \"739a7635-b32c-4f38-8193-2975abc5e092\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:40:33.871209807Z\",\n \"endTime\": \"2026-08-21T02:59:33.678476291Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f969ddc66388f6636eb6fb562e07c82ce37d6c1699beba95e7c983471e1cd1bf","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, kube-scheduler, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60b255e30748c8a6d90671575f6b11ca13ea12cf92cc1ad0eb278b280f54ddb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"684f37cf48d101c235cdeb5cebaf623acfb01e63bf09a5dc4ed857bd9fa46e14","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edcea0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8fl8dx2bw22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\",\n \"name\": \"42d6143a-e97b-4483-afd9-f74bce1b7086\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:44:46.647452675Z\",\n \"endTime\": \"2026-08-21T03:03:54.737093461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edcea0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8fl8dx2bw22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\",\n \"name\": \"42d6143a-e97b-4483-afd9-f74bce1b7086\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:44:46.647452675Z\",\n \"endTime\": \"2026-08-21T03:03:54.737093461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"626c13aba65b7887cb770b9e292390fc9ce6f12c6f6522432ec65fd5683a9617","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"b3374a6a172ebb403ac686e50368e308458096eea2258fdb839e7b2281ad2231","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-dqp9mm5cksqn/cluster-candidate-4-20-bfprqt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015cf400\u003e: \n failed to create HCP cluster cluster-candidate-4-20-bfprqt: failed waiting for cluster=\"cluster-candidate-4-20-bfprqt\" in resourcegroup=\"rg-candidate-4-20-dqp9mm5cksqn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n \"name\": \"e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:18.115131874Z\",\n \"endTime\": \"2026-08-21T02:22:22.541705225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-dqp9mm5cksqn/cluster-candidate-4-20-bfprqt should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015cf400\u003e: \n failed to create HCP cluster cluster-candidate-4-20-bfprqt: failed waiting for cluster=\"cluster-candidate-4-20-bfprqt\" in resourcegroup=\"rg-candidate-4-20-dqp9mm5cksqn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n \"name\": \"e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:18.115131874Z\",\n \"endTime\": \"2026-08-21T02:22:22.541705225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b51c13a3cafe6196693705cdaea38d26216b0eac963a598136ad9cacb24071b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"724a508ec717e9dfedf1d20efce653e54ed5fa958c79619a4c4224db1a9a7d55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"b84e75c4953bfe9005b85e725e144df6a003f1fa4344d9a74c269a1620ea8307","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-4pwkcd\" with version 4.20.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a42900\u003e: \n failed to create HCP cluster cluster-zstream-4-20-4pwkcd: failed waiting for cluster=\"cluster-zstream-4-20-4pwkcd\" in resourcegroup=\"rg-zstream-upgrade-4-20-lmtg4sqfvqsl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0fadd9-5dbd-4358-9cdd-7b73d96b6490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0fadd9-5dbd-4358-9cdd-7b73d96b6490\",\n \"name\": \"2c0fadd9-5dbd-4358-9cdd-7b73d96b6490\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:37.313877591Z\",\n \"endTime\": \"2026-08-21T02:51:43.387564668Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-4pwkcd\" with version 4.20.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a42900\u003e: \n failed to create HCP cluster cluster-zstream-4-20-4pwkcd: failed waiting for cluster=\"cluster-zstream-4-20-4pwkcd\" in resourcegroup=\"rg-zstream-upgrade-4-20-lmtg4sqfvqsl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0fadd9-5dbd-4358-9cdd-7b73d96b6490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0fadd9-5dbd-4358-9cdd-7b73d96b6490\",\n \"name\": \"2c0fadd9-5dbd-4358-9cdd-7b73d96b6490\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:37.313877591Z\",\n \"endTime\": \"2026-08-21T02:51:43.387564668Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"871a5da5a24d3bc727297b748aea69d61d079631295a834c16d1075d6f918955","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"f2ec5a1a25d21e5dec2ed93eab92a26a91c080ad53e104d71fabc59db7c4d615","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e401e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tm5ghkxjvgwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d777726a-9d19-4bf4-94d3-fe0b95950e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d777726a-9d19-4bf4-94d3-fe0b95950e3c\",\n \"name\": \"d777726a-9d19-4bf4-94d3-fe0b95950e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:36.864905839Z\",\n \"endTime\": \"2026-08-21T02:22:42.517743708Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e401e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tm5ghkxjvgwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d777726a-9d19-4bf4-94d3-fe0b95950e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d777726a-9d19-4bf4-94d3-fe0b95950e3c\",\n \"name\": \"d777726a-9d19-4bf4-94d3-fe0b95950e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:36.864905839Z\",\n \"endTime\": \"2026-08-21T02:22:42.517743708Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"878c5311540df1bc33c2bc2303f199fcb24031eb06d539442b32d0462d78cc97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"7ddd8e0bddb95c4a8ef63e85e6c7889f6f625f289311cf197d6695415a140810","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048e40\u003e: \n failed to create HCP cluster np-dg-minor-dc29s7: failed waiting for cluster=\"np-dg-minor-dc29s7\" in resourcegroup=\"rg-np-dg-minor-dc29s7-9gmklftjh28x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n \"name\": \"6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:49:33.691417488Z\",\n \"endTime\": \"2026-08-21T03:08:34.727253217Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048e40\u003e: \n failed to create HCP cluster np-dg-minor-dc29s7: failed waiting for cluster=\"np-dg-minor-dc29s7\" in resourcegroup=\"rg-np-dg-minor-dc29s7-9gmklftjh28x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n \"name\": \"6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:49:33.691417488Z\",\n \"endTime\": \"2026-08-21T03:08:34.727253217Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c94f79fadf5f0c5900ad5aba81751610a3b4c431f48765be0c4ce0b0f37bd7e9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"885ab1cee254586adbee64a06ad3711ee48c4b9e89d72ba8710e635c653c1a84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"ff8b79ab35f4ee1da283d000f55c6cb2c5767888f63ecfe86c10a751852965b5","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5lt48h with version 4.21.29\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068a60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-5lt48h: failed waiting for cluster=\"np-version-upgrade-cluster-5lt48h\" in resourcegroup=\"rg-np-version-upgrade-5lt48h-dszsrsbrq6zk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n \"name\": \"fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:45:36.662949608Z\",\n \"endTime\": \"2026-08-21T03:04:44.745401761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.q5e3m9a0x8h7s0u.7eq2.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5lt48h with version 4.21.29\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068a60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-5lt48h: failed waiting for cluster=\"np-version-upgrade-cluster-5lt48h\" in resourcegroup=\"rg-np-version-upgrade-5lt48h-dszsrsbrq6zk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n \"name\": \"fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:45:36.662949608Z\",\n \"endTime\": \"2026-08-21T03:04:44.745401761Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.q5e3m9a0x8h7s0u.7eq2.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc30c2cd1479d681b3c8e85dbe87f378840f2bf26b28373ee01d7b48d759e41c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, konnectivity-agent, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"95ab5bd94bf27a79467d33bdf9e7c4450c228d5fc74f7480a21eec13572ba6fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"021ad8856d5c396fba67aba9c2c2ff60d51660b492523606a295b59e5f5dfb89","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-h762z2 with version 4.21.29\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000916f80\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-h762z2: failed waiting for cluster=\"np-version-upgrade-cluster-h762z2\" in resourcegroup=\"rg-np-version-upgrade-h762z2-227jm54s5fnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n \"name\": \"7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:45:47.941159859Z\",\n \"endTime\": \"2026-08-21T03:04:54.892784877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-h762z2 with version 4.21.29\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000916f80\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-h762z2: failed waiting for cluster=\"np-version-upgrade-cluster-h762z2\" in resourcegroup=\"rg-np-version-upgrade-h762z2-227jm54s5fnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n \"name\": \"7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:45:47.941159859Z\",\n \"endTime\": \"2026-08-21T03:04:54.892784877Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bae2c31dfd388275f409339683e6035aa6871992170a42fd34eae18131b6e2ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"987d7e60eddff5ea77c16b40cb170ea1f2bac7558196ff3d1adf6166b2e90ba3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"dc7a3b65992904f6bd95eec51bb94c8945852f4be0b05c741ee7f46e3bc1a620","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001178800\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-njrg4blgvtnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\",\n \"name\": \"5481864c-7cad-43c3-99c6-7f83d731f12b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:29.121350695Z\",\n \"endTime\": \"2026-08-21T02:22:32.49197336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001178800\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-njrg4blgvtnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\",\n \"name\": \"5481864c-7cad-43c3-99c6-7f83d731f12b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:29.121350695Z\",\n \"endTime\": \"2026-08-21T02:22:32.49197336Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96ac8f4694917334827139385bf0e43172c70aa0ffc95a888499b14a4f14ff9d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a72dafcc2d3ff6b1ce95a79de907a64dced78404692ba2f3a6041f4ef042fa3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"8bf1efa5a500673bb5d80c68b86c9d236a42c04862fb84b7d85434272615d259","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-stz97h\" with version 4.21.28 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005b4620\u003e: \n failed to create HCP cluster cluster-zstream-4-21-stz97h: failed waiting for cluster=\"cluster-zstream-4-21-stz97h\" in resourcegroup=\"rg-zstream-upgrade-4-21-jr9rvssp6jdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n \"name\": \"dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:46.784949272Z\",\n \"endTime\": \"2026-08-21T02:51:53.64362497Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, cluster-version-operator, cluster-policy-controller, dns-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, olm-operator, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, kube-controller-manager, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-stz97h\" with version 4.21.28 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005b4620\u003e: \n failed to create HCP cluster cluster-zstream-4-21-stz97h: failed waiting for cluster=\"cluster-zstream-4-21-stz97h\" in resourcegroup=\"rg-zstream-upgrade-4-21-jr9rvssp6jdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n \"name\": \"dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:32:46.784949272Z\",\n \"endTime\": \"2026-08-21T02:51:53.64362497Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, cluster-version-operator, cluster-policy-controller, dns-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, olm-operator, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, kube-controller-manager, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4c7965c412b351e80ef0544bff250a8a694f957336d67b40d995be9b67f72d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b3764036b7c1a3b9c135ca667e02dd505b1aca7dffcefcf936af2764e021ed1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"b0b84b753e0f678ce469ee4fc8400cad30e638a19fedbd6f7746aad6ef378b5a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a90960\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-c7xznlxpmm8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fce9b8c6-a006-4ff2-ade7-cbcb7972be94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fce9b8c6-a006-4ff2-ade7-cbcb7972be94\",\n \"name\": \"fce9b8c6-a006-4ff2-ade7-cbcb7972be94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T03:04:39.108988672Z\",\n \"endTime\": \"2026-08-21T03:23:44.786170105Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a90960\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-c7xznlxpmm8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fce9b8c6-a006-4ff2-ade7-cbcb7972be94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fce9b8c6-a006-4ff2-ade7-cbcb7972be94\",\n \"name\": \"fce9b8c6-a006-4ff2-ade7-cbcb7972be94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T03:04:39.108988672Z\",\n \"endTime\": \"2026-08-21T03:23:44.786170105Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a3cbbc950febd785fe4ca267276312fb3ebc4f952975a3457d1434f0c082d23c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"d06b61b2a290f31b2b44796608e3a50ed66dd83e5cff65620f4a4dbd778567f6","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"aa82a70d0f822af59237c40244b72302570a35d1980c7239e4d1fea44c0a545f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"8239e5a1956b5200678e34e906157e9a5f4c31e6442b526e2f03f3fb5593beb2","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001344800\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-q78wj4tcrbbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n \"name\": \"31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:44:35.391627616Z\",\n \"endTime\": \"2026-08-21T03:03:43.616320763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001344800\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-q78wj4tcrbbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n \"name\": \"31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:44:35.391627616Z\",\n \"endTime\": \"2026-08-21T03:03:43.616320763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"84c9b6f5f7cc00976416d8874ac20f25a776a6f9140090905fa5d8c5a508047b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ace5d74c5a7b8892611d7f3c8bb5b2acedd0ebe137b701efc3b4287cc70f8ff1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"7fb8dc9dd6caa75c82f132d73240ca1b73d2c2ed70d78cc29ea092d7fc3ce43b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a6420\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wpsgmw5rph44\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n \"name\": \"eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:05:11.172903386Z\",\n \"endTime\": \"2026-08-21T02:24:12.542645954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a6420\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wpsgmw5rph44\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n \"name\": \"eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:05:11.172903386Z\",\n \"endTime\": \"2026-08-21T02:24:12.542645954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90cdb36cc9a54567f53aaac3f8c671c2433ddd6b58320efddb4af3f213c50608","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ae7133f57c5f7b1ffe624e835769bc2694bef7955a51f3a83611e6182dad1901","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"238e7b24b648c5d6d6127862df062515aa947e0f889da3e18d7d7c8417d0316a","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0f080\u003e: \n failed to create HCP cluster np-downgrade-kjw6pt: failed waiting for cluster=\"np-downgrade-kjw6pt\" in resourcegroup=\"rg-np-downgrade-kjw6pt-lf69f9frnjjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\",\n \"name\": \"98f96261-7c0c-4604-82d2-ccf83d040db2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:48:21.62730862Z\",\n \"endTime\": \"2026-08-21T03:07:24.73168706Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, cluster-policy-controller, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, konnectivity-agent, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, packageserver, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, openshift-apiserver, cluster-version-operator, dns-operator, olm-operator, olm-collect-profiles, kube-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0f080\u003e: \n failed to create HCP cluster np-downgrade-kjw6pt: failed waiting for cluster=\"np-downgrade-kjw6pt\" in resourcegroup=\"rg-np-downgrade-kjw6pt-lf69f9frnjjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\",\n \"name\": \"98f96261-7c0c-4604-82d2-ccf83d040db2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:48:21.62730862Z\",\n \"endTime\": \"2026-08-21T03:07:24.73168706Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, cluster-policy-controller, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, konnectivity-agent, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, packageserver, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, openshift-apiserver, cluster-version-operator, dns-operator, olm-operator, olm-collect-profiles, kube-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b7edb3184e83c8788b81e7260687887dbb6c35194bd95d24f983682b926feef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"91e88f33d22326cd1e1f28e5cf546d8a5f825f85b45e6974877ac8761f301bbc","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005fe2a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ttg7gz2lhlcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\",\n \"name\": \"ba59ea70-9b35-4783-a0ec-868e44666cde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:50:23.471170015Z\",\n \"endTime\": \"2026-08-21T03:09:24.769917396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, dns-operator, catalog-operator, cluster-image-registry-operator, community-operators-catalog, openshift-route-controller-manager, olm-operator, cluster-policy-controller, certified-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005fe2a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ttg7gz2lhlcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\",\n \"name\": \"ba59ea70-9b35-4783-a0ec-868e44666cde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:50:23.471170015Z\",\n \"endTime\": \"2026-08-21T03:09:24.769917396Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, dns-operator, catalog-operator, cluster-image-registry-operator, community-operators-catalog, openshift-route-controller-manager, olm-operator, cluster-policy-controller, certified-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6fc45abab45bac910821b9bf649e0a8d2f5cae8a2886fe2e7857c4e0907eb987","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bcba71bf448b7c2acdfa9b0daf4e48520a178306a8de4600f2e11ee32e9cce30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"2b76ccfe8aac8f29923c772019c68b4d1daa538e7e3fedec8fb0b2f71b8b486f","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-5jbtkc\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00147aa60\u003e: \n failed to create HCP cluster np-noedge-5jbtkc: failed waiting for cluster=\"np-noedge-5jbtkc\" in resourcegroup=\"rg-np-noedge-5jbtkc-5kwghxcpjkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\",\n \"name\": \"edc3030e-e575-42e1-8662-b09b51002c15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:45:54.462061981Z\",\n \"endTime\": \"2026-08-21T03:04:54.752602351Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, dns-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, cluster-policy-controller, certified-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-network-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-5jbtkc\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00147aa60\u003e: \n failed to create HCP cluster np-noedge-5jbtkc: failed waiting for cluster=\"np-noedge-5jbtkc\" in resourcegroup=\"rg-np-noedge-5jbtkc-5kwghxcpjkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\",\n \"name\": \"edc3030e-e575-42e1-8662-b09b51002c15\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:45:54.462061981Z\",\n \"endTime\": \"2026-08-21T03:04:54.752602351Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, dns-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, cluster-policy-controller, certified-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-network-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6fc45abab45bac910821b9bf649e0a8d2f5cae8a2886fe2e7857c4e0907eb987","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bcff38b1f6f8e11825a4aa3c7b9e9cc52a7a013adcb0f9ee0762872425e88493","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"565fce7fcb4445a79f30034d53708e522a56029627f0a6e682c9320663ffa91f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-wlltck9mnpdh/cluster-candidate-4-21-wqbh9c should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0c460\u003e: \n failed to create HCP cluster cluster-candidate-4-21-wqbh9c: failed waiting for cluster=\"cluster-candidate-4-21-wqbh9c\" in resourcegroup=\"rg-candidate-4-21-wlltck9mnpdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\",\n \"name\": \"61fe2608-61d1-400b-90c7-806bb5489f2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:28.583859524Z\",\n \"endTime\": \"2026-08-21T02:22:32.536144456Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-wlltck9mnpdh/cluster-candidate-4-21-wqbh9c should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0c460\u003e: \n failed to create HCP cluster cluster-candidate-4-21-wqbh9c: failed waiting for cluster=\"cluster-candidate-4-21-wqbh9c\" in resourcegroup=\"rg-candidate-4-21-wlltck9mnpdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\",\n \"name\": \"61fe2608-61d1-400b-90c7-806bb5489f2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:28.583859524Z\",\n \"endTime\": \"2026-08-21T02:22:32.536144456Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd3de68230f91b2c05b7364b0fa8cf02cc16770e1eeacb4b4a40826ba04c3a0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"81fc71e1b2add474a2f9142629a0d6332c8b66a31ce3ed781eff6cdd129c1fb1","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006fd680\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pjrkd5gdglbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00cc9a83-4a83-44e1-93f2-af3745f7d728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00cc9a83-4a83-44e1-93f2-af3745f7d728\",\n \"name\": \"00cc9a83-4a83-44e1-93f2-af3745f7d728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:28.849632387Z\",\n \"endTime\": \"2026-08-21T02:22:32.506414086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006fd680\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pjrkd5gdglbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00cc9a83-4a83-44e1-93f2-af3745f7d728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00cc9a83-4a83-44e1-93f2-af3745f7d728\",\n \"name\": \"00cc9a83-4a83-44e1-93f2-af3745f7d728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:28.849632387Z\",\n \"endTime\": \"2026-08-21T02:22:32.506414086Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cf0529bf104127949c20835a88beb727ff943b348fa814fbdcdef9a5b08835c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"32a0bd18972a8f8e83478af297e9368419e1b7e7597a308653070e30fe4fe6c7","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc300\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dnkv5vg5t2ts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\",\n \"name\": \"abfca9a0-da31-4fda-b740-f92e856a7dca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:36.161779463Z\",\n \"endTime\": \"2026-08-21T02:22:42.321981114Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-node-tuning-operator, csi-snapshot-controller-operator, ignition-server-proxy, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, olm-collect-profiles, redhat-marketplace-catalog, certified-operators-catalog, packageserver, community-operators-catalog, machine-approver, control-plane-pki-operator, ignition-server, openshift-controller-manager, dns-operator, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, openshift-apiserver, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc300\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dnkv5vg5t2ts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\",\n \"name\": \"abfca9a0-da31-4fda-b740-f92e856a7dca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:36.161779463Z\",\n \"endTime\": \"2026-08-21T02:22:42.321981114Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-node-tuning-operator, csi-snapshot-controller-operator, ignition-server-proxy, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, olm-collect-profiles, redhat-marketplace-catalog, certified-operators-catalog, packageserver, community-operators-catalog, machine-approver, control-plane-pki-operator, ignition-server, openshift-controller-manager, dns-operator, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, openshift-apiserver, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d2bd7499b54104b3d3d24bd3740ffad953111d50dc8c4f7bff2b6ec10a4a0d7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"854ad62699d22398160556ff1d1461a048fa03f78571ec0dcf40580c9a2fd02d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-m6dffq9dg6px/cluster-candidate-4-22-rttbvz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fee820\u003e: \n failed to create HCP cluster cluster-candidate-4-22-rttbvz: failed waiting for cluster=\"cluster-candidate-4-22-rttbvz\" in resourcegroup=\"rg-candidate-4-22-m6dffq9dg6px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\",\n \"name\": \"6621367e-101e-4d5a-ab22-9e61d95865dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:29:22.397457374Z\",\n \"endTime\": \"2026-08-21T02:48:23.657141881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-m6dffq9dg6px/cluster-candidate-4-22-rttbvz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fee820\u003e: \n failed to create HCP cluster cluster-candidate-4-22-rttbvz: failed waiting for cluster=\"cluster-candidate-4-22-rttbvz\" in resourcegroup=\"rg-candidate-4-22-m6dffq9dg6px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\",\n \"name\": \"6621367e-101e-4d5a-ab22-9e61d95865dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:29:22.397457374Z\",\n \"endTime\": \"2026-08-21T02:48:23.657141881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b51c13a3cafe6196693705cdaea38d26216b0eac963a598136ad9cacb24071b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de89b7276630b7f0726401547c827daddbbac11200689c38256c6b6f5bb4017a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"0ed7bae60681c60f68c493d90cb1fd137349d28e37ec331f94e67f0f0671c10a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xbp8kt\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d57060\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-xbp8kt: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xbp8kt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-mvbldsj9x64s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n \"name\": \"d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:33:00.457364861Z\",\n \"endTime\": \"2026-08-21T02:52:03.680608781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, control-plane-pki-operator, kube-scheduler, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, ignition-server, olm-operator, machine-approver, olm-collect-profiles, openshift-apiserver, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, packageserver, community-operators-catalog, cluster-autoscaler, cluster-version-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xbp8kt\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d57060\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-xbp8kt: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xbp8kt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-mvbldsj9x64s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n \"name\": \"d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:33:00.457364861Z\",\n \"endTime\": \"2026-08-21T02:52:03.680608781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, control-plane-pki-operator, kube-scheduler, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, ignition-server, olm-operator, machine-approver, olm-collect-profiles, openshift-apiserver, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, packageserver, community-operators-catalog, cluster-autoscaler, cluster-version-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec60ce3f4f9a623a70a6fc7c12c157c06874562235d306084e53eeacbde3b300","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"6a04db7e50f70ea9dac9932acea60a13f6c9b7d4f8617adb14a39552f096a365","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082e8c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f4r2j84gbzgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a18db25-ebe7-4568-9017-f02c59786f68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a18db25-ebe7-4568-9017-f02c59786f68\",\n \"name\": \"7a18db25-ebe7-4568-9017-f02c59786f68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:04:14.557262293Z\",\n \"endTime\": \"2026-08-21T02:23:22.701996424Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082e8c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-f4r2j84gbzgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a18db25-ebe7-4568-9017-f02c59786f68\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a18db25-ebe7-4568-9017-f02c59786f68\",\n \"name\": \"7a18db25-ebe7-4568-9017-f02c59786f68\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:04:14.557262293Z\",\n \"endTime\": \"2026-08-21T02:23:22.701996424Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f68f576849e3b31740d4f379af3cb11457d90bb2868efbad74c631f2e8ed9f0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"60abba32a30fc314bf3f1591fe14612458a80c23ca459244b846d1098cad1d5d","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce5680\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s6nq8d7z6854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n \"name\": \"68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:04:33.015430331Z\",\n \"endTime\": \"2026-08-21T02:23:42.504347676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce5680\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s6nq8d7z6854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n \"name\": \"68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:04:33.015430331Z\",\n \"endTime\": \"2026-08-21T02:23:42.504347676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6e3b20deb9bae580ddd8091aa6a577e844f6b83692c9312152315f89fed55d3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f897986d062f1ff74cef089d98aa2789cf4a18f22c9c9ee845e7cfc95d9890c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"37d9e7ef301047c2b2fd4899c663861930478c9a873b27090cbd860df00a8a91","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010221c0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8p587dnw2vfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e222a20f-7ada-4ebc-a6d6-2cbf5af1944f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e222a20f-7ada-4ebc-a6d6-2cbf5af1944f\",\n \"name\": \"e222a20f-7ada-4ebc-a6d6-2cbf5af1944f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:38.157530517Z\",\n \"endTime\": \"2026-08-21T02:22:42.506786344Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, ignition-server-proxy, kube-controller-manager, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, cluster-image-registry-operator, ignition-server, redhat-marketplace-catalog, olm-operator, olm-collect-profiles, cluster-autoscaler, openshift-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, community-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, konnectivity-agent, catalog-operator, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010221c0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8p587dnw2vfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e222a20f-7ada-4ebc-a6d6-2cbf5af1944f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e222a20f-7ada-4ebc-a6d6-2cbf5af1944f\",\n \"name\": \"e222a20f-7ada-4ebc-a6d6-2cbf5af1944f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:03:38.157530517Z\",\n \"endTime\": \"2026-08-21T02:22:42.506786344Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, cluster-policy-controller, ignition-server-proxy, kube-controller-manager, csi-snapshot-controller-operator, redhat-operators-catalog, certified-operators-catalog, machine-approver, kube-scheduler, cluster-version-operator, dns-operator, cluster-image-registry-operator, ignition-server, redhat-marketplace-catalog, olm-operator, olm-collect-profiles, cluster-autoscaler, openshift-controller-manager, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, packageserver, community-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-network-operator, konnectivity-agent, catalog-operator, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f9912df6e42e2ada7213ae0f22b9c69a6d573965d601c11f0b374f5226eb1c89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","failed_at":"e2e","signature_id":"950772107db22c8db323365cbb8dc7ad54495494533bdb154fc48dbef42f844b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d11040\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-b5ncfgjg8jnm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48765129-507a-498b-b68a-9bb0953bd609\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48765129-507a-498b-b68a-9bb0953bd609\",\n \"name\": \"48765129-507a-498b-b68a-9bb0953bd609\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:35:14.820044946Z\",\n \"endTime\": \"2026-08-21T02:54:23.666182005Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, catalog-operator, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-node-tuning-operator, kube-controller-manager, cluster-version-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, cluster-storage-operator, ingress-operator, konnectivity-agent, ignition-server, ignition-server-proxy, cluster-autoscaler, openshift-controller-manager, hosted-cluster-config-operator, dns-operator, olm-collect-profiles, machine-approver, openshift-apiserver, cluster-network-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d11040\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-b5ncfgjg8jnm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48765129-507a-498b-b68a-9bb0953bd609\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48765129-507a-498b-b68a-9bb0953bd609\",\n \"name\": \"48765129-507a-498b-b68a-9bb0953bd609\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-21T02:35:14.820044946Z\",\n \"endTime\": \"2026-08-21T02:54:23.666182005Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: control-plane-pki-operator, catalog-operator, kube-scheduler, azure-cloud-controller-manager, cluster-image-registry-operator, redhat-marketplace-catalog, openshift-route-controller-manager, cluster-node-tuning-operator, kube-controller-manager, cluster-version-operator, olm-operator, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog, cluster-policy-controller, cluster-storage-operator, ingress-operator, konnectivity-agent, ignition-server, ignition-server-proxy, cluster-autoscaler, openshift-controller-manager, hosted-cluster-config-operator, dns-operator, olm-collect-profiles, machine-approver, openshift-apiserver, cluster-network-operator, packageserver, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":43,"failure_pattern_ids":["17cbac65ad6dde679afa9f8374f5b421e45ae642715ea520e35f5231ee1e4b90","3b51c13a3cafe6196693705cdaea38d26216b0eac963a598136ad9cacb24071b","3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","6cef856251578d6d4f842b0ea51927280752396355aaa78eb5a3ffb310ef3d92","6fc45abab45bac910821b9bf649e0a8d2f5cae8a2886fe2e7857c4e0907eb987","819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","84c9b6f5f7cc00976416d8874ac20f25a776a6f9140090905fa5d8c5a508047b","90cdb36cc9a54567f53aaac3f8c671c2433ddd6b58320efddb4af3f213c50608","96ac8f4694917334827139385bf0e43172c70aa0ffc95a888499b14a4f14ff9d","a4c7965c412b351e80ef0544bff250a8a694f957336d67b40d995be9b67f72d9","acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","bae2c31dfd388275f409339683e6035aa6871992170a42fd34eae18131b6e2ff","c6e3b20deb9bae580ddd8091aa6a577e844f6b83692c9312152315f89fed55d3","c94f79fadf5f0c5900ad5aba81751610a3b4c431f48765be0c4ce0b0f37bd7e9","d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","dc30c2cd1479d681b3c8e85dbe87f378840f2bf26b28373ee01d7b48d759e41c","f969ddc66388f6636eb6fb562e07c82ce37d6c1699beba95e7c983471e1cd1bf","fcefb659555e6a990fedc929fc65e2b32862b237e18ebc38834dc9a0eaf288eb"],"matched_occurrences":43,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}