{"meta":{"start_date":"2026-08-20","end_date":"2026-08-20","start_at":"2026-08-20T00:00:00Z","end_at":"2026-08-21T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T22:16:01Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":38,"matched_occurrences":38,"runs_affected":6},"rows":[{"environment":"stg","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,4,4],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000895308\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list7lw9mf\" in resourcegroup=\"ea-list-2lm6qlj6qrb9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"name\": \"fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:55.575829675Z\",\n      \"endTime\": \"2026-08-20T10:09:02.513171982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7keqbo3pl38n1vug7bjfptpeikrn1 still exists (deletion dispatched at 2026-08-20T09:44:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000895308\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list7lw9mf\" in resourcegroup=\"ea-list-2lm6qlj6qrb9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"name\": \"fe059e7f-9879-4a59-879c-7c2d634a7dc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:55.575829675Z\",\n      \"endTime\": \"2026-08-20T10:09:02.513171982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7keqbo3pl38n1vug7bjfptpeikrn1 still exists (deletion dispatched at 2026-08-20T09:44:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001084138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9zcvtfmqbq5k\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"name\": \"d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:46:19.133487927Z\",\n      \"endTime\": \"2026-08-20T10:10:22.487349206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kgqae0lr73vi3huktro85eu2akrr still exists (deletion dispatched at 2026-08-20T09:46:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001084138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9zcvtfmqbq5k\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"name\": \"d5a33047-0436-48bd-99bc-1757d78738bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:46:19.133487927Z\",\n      \"endTime\": \"2026-08-20T10:10:22.487349206Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kgqae0lr73vi3huktro85eu2akrr still exists (deletion dispatched at 2026-08-20T09:46:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"c98eeb6d2064c80a5c88dac50a9f84605cf8ee1eb70a70345435f7284fcc94c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"0facf4aca2cbbc09d1d4d2b5f2a2c443dc9ac7bce839a50cfc50190997e06fd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"a1399ba06ca55ef9ac4602a99872b2b1d13798c7ec52bf26477719dd69c14e17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"d19ec4d4f31d760d02b50b389f562a65f23856356a3f633669a10c48565d949d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":147,"trend_counts":[34,34,34,2,0,0,4],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":11,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b026c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-447nnn7lv7ss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"name\": \"68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:54:41.378616139Z\",\n      \"endTime\": \"2026-08-20T09:13:49.738147219Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b026c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-447nnn7lv7ss\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"name\": \"68bcf8b9-c7f6-4e6f-9f16-24314ebf9371\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:54:41.378616139Z\",\n      \"endTime\": \"2026-08-20T09:13:49.738147219Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000caeac0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lfvfr4f2rlsg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"name\": \"f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:43:33.944124188Z\",\n      \"endTime\": \"2026-08-20T10:02:41.325901249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000caeac0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-lfvfr4f2rlsg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"name\": \"f18bc740-6f82-415a-81e5-6d3a6a94b912\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:43:33.944124188Z\",\n      \"endTime\": \"2026-08-20T10:02:41.325901249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090538506672672768","occurred_at":"2026-08-20T20:36:34Z","signature_id":"75e8134db7c38c2272b0135b9ef67b8e4ba756be9b98292cf44f0135c9355207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"56f9bbbf6c1cff5e3a43f7b42723ff5d2896274197334b06141ae6c0de7f3f12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"7ebf3a7566f82ffac1f2fdc94973962a97b9cebbccee3a25837ecc58e0903fb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"f2f000bdc64541602873ebc887a909036d077a88b2e4d46c3fb4c2246d3285bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe3060\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8vkz6566fxqz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"name\": \"140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:33.753602969Z\",\n      \"endTime\": \"2026-08-20T13:35:40.453844587Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-autoscaler, cluster-node-tuning-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, azure-cloud-controller-manager, cluster-network-operator, packageserver, olm-collect-profiles, redhat-marketplace-catalog, openshift-apiserver, dns-operator, catalog-operator, olm-operator, cluster-policy-controller, cluster-storage-operator, machine-approver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent, kube-storage-version-migrator, kube-scheduler, openshift-route-controller-manager, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe3060\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8vkz6566fxqz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"name\": \"140c4b91-4a5b-4a44-a63d-5f39fc772c32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:33.753602969Z\",\n      \"endTime\": \"2026-08-20T13:35:40.453844587Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, csi-snapshot-controller-operator, community-operators-catalog, cluster-autoscaler, cluster-node-tuning-operator, redhat-operators-catalog, certified-operators-catalog, kube-controller-manager, azure-cloud-controller-manager, cluster-network-operator, packageserver, olm-collect-profiles, redhat-marketplace-catalog, openshift-apiserver, dns-operator, catalog-operator, olm-operator, cluster-policy-controller, cluster-storage-operator, machine-approver, openshift-controller-manager, hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, konnectivity-agent, kube-storage-version-migrator, kube-scheduler, openshift-route-controller-manager, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc4980\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2qbp55nw5hq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"name\": \"3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:17:04.908245766Z\",\n      \"endTime\": \"2026-08-20T13:36:10.801854971Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, konnectivity-agent, catalog-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-controller-manager, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, olm-operator, kube-scheduler, dns-operator, ingress-operator, csi-snapshot-controller-operator, ignition-server-proxy, certified-operators-catalog, cluster-storage-operator, ignition-server, control-plane-pki-operator, olm-collect-profiles, openshift-route-controller-manager, cluster-network-operator, cluster-image-registry-operator, machine-approver, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-autoscaler, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc4980\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2qbp55nw5hq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"name\": \"3cab7363-1a89-4114-8a30-15cd32af7525\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:17:04.908245766Z\",\n      \"endTime\": \"2026-08-20T13:36:10.801854971Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, konnectivity-agent, catalog-operator, packageserver, redhat-operators-catalog, community-operators-catalog, kube-controller-manager, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, azure-cloud-controller-manager, olm-operator, kube-scheduler, dns-operator, ingress-operator, csi-snapshot-controller-operator, ignition-server-proxy, certified-operators-catalog, cluster-storage-operator, ignition-server, control-plane-pki-operator, olm-collect-profiles, openshift-route-controller-manager, cluster-network-operator, cluster-image-registry-operator, machine-approver, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-autoscaler, openshift-apiserver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6ed60\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-vp6x5vhlr7vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"name\": \"0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:30:45.72149794Z\",\n      \"endTime\": \"2026-08-20T09:49:51.406315863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server-proxy, dns-operator, ingress-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, openshift-controller-manager, openshift-route-controller-manager, packageserver, olm-collect-profiles, machine-approver, olm-operator, control-plane-pki-operator, openshift-apiserver, cluster-policy-controller, hosted-cluster-config-operator, redhat-operators-catalog, community-operators-catalog, catalog-operator, ignition-server, certified-operators-catalog, konnectivity-agent, cluster-network-operator, cluster-autoscaler; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6ed60\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-vp6x5vhlr7vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"name\": \"0187c1e1-0a74-4fa5-a1a0-3c36f116fe55\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:30:45.72149794Z\",\n      \"endTime\": \"2026-08-20T09:49:51.406315863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ignition-server-proxy, dns-operator, ingress-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, kube-scheduler, cluster-node-tuning-operator, redhat-marketplace-catalog, kube-controller-manager, cluster-version-operator, azure-cloud-controller-manager, cluster-storage-operator, openshift-controller-manager, openshift-route-controller-manager, packageserver, olm-collect-profiles, machine-approver, olm-operator, control-plane-pki-operator, openshift-apiserver, cluster-policy-controller, hosted-cluster-config-operator, redhat-operators-catalog, community-operators-catalog, catalog-operator, ignition-server, certified-operators-catalog, konnectivity-agent, cluster-network-operator, cluster-autoscaler; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"fd1a222e88fac51899884139c985b18a14bda354bbdc6d03b6bd0f1ec3b9916a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"fd7348293825c9d41446d338b7d31ffd23035dfeb87dc5ea60ab54b00dd1a82a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"f86b3413a4fa89f0fe3aee0a97c3b3aae586cbd24a62a162177322eec5c926c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"1fe08479e59e5ed4afa0e1f161e992bf328632b651a998809d33c2ece866d07c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","signature_id":"7e2cf5d1bff2be054d231178fc2524db6c79b0da8fc042c6972e38c1b9001d27","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[1,0,0,0,0,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T02:00:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"382e53c6f5440687f3efd2f0f735d848f4f2a75c27dbddc1aa370d48842e5ac7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"3aeb591de359ede5d82e500a19257072d85b9193e57ac8f4a13b125e2c1b0f69","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":12,"trend_counts":[0,0,0,4,1,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":11,"prior_last_seen_at":"2026-08-04T17:43:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"30e3ce44a70a32080f494a46962fb9e3504a602768cbec5e3cc893e94e06d32f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","signature_id":"e14f3f9d9bbacbaff6e558f257e54562c56eab915ed893444a9648f95512b4c2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-08-03","2026-08-10"],"prior_runs_affected":24,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027ec2a0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-cxh78blbgmpq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027ec2a0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-cxh78blbgmpq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-cxh78blbgmpq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dgkn5p9l9flc/cluster-candidate-5-0-nrmbvd should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a85590\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dgkn5p9l9flc/cluster-candidate-5-0-nrmbvd should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a85590\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"5b48ea80fc398a034dba3d46b19e592386ab5e5708b0a0e23267c0282b5a55be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"2c025bacef2a95219db5cc2c1d0f8e25156096d570a86e526e43447bf8f9a29b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-20","2026-07-27"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-01T23:00:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0025d2270\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7lgrn.apps.aro.agg-cluster.zfzg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0025d2270\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-7lgrn.apps.aro.agg-cluster.zfzg.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000de51e8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2lhgp.apps.aro.agg-cluster.v7n3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000de51e8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2lhgp.apps.aro.agg-cluster.v7n3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"4649980ccb572a8df4fef4e44831f1de4eae2d176d8ecffa92fb99f1ae87aa7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"bf90b8b00390291912ae577c4eaa041b4552832f8c625cf2cd2cff628c244b5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,3,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-06T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"3f5ac4cdd7ff19e9ff918b763fc044b6a568c83bba923b1245e9a1c9566d493d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"e0f6f930a179a3343bf2fd68ac2daa5ec05546590115b913738327ca0e47fe4d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"bef58880af3f67335fb38c9962fdc80b164c51531808fd3a944b58b5abc7835e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"fdab187cfe558b4bb3f41aa7470064acf21835a87bb5df9e4d6a394f761d1b45","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f170a7927803930d63ef797276984cf62fcbd2f07de426245306d08da19e2bb9","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"ee2ed8274c95a8959d110687ae9ece838bb447f44f7c22e5c06d8ad3a3a7f27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"ee48f986fd9e592059d2a5cd09e88c38d974e8948f1e0f0065c5a8641bfef3cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d8e13d55daacf4ddf74bdc040314e449b308767e96d40780e45800466f812928","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e","search_query":"list nodes (nodePool=npdg-422-420): Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"3519eaf3902036b2df340ad7c69439459fca96026f79aacac0e4ada2e8cb9ab4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"846c6382df9acef0fa5ddc2a6d1f4dc3f653f6d1f1033da3dc950698cda0412f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7356cf97f840631ab4a0a63ce4f36ca842474a0ca4160d56ce50ad927de67954","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"50cf928a4b9ab91ad463923b6c69c34145bea77427cbfe46d31360d594882c1c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6722502d5d665731952f221ae4eff4c1fe60411f08b65ee3c8c1ea7fe37c4a2f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"37da2bd93766d6d4d4f6961c9e8500becdfb67c804e717112af85f0368264347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4b6bc70dde3370d5e666a11dcdf8b092a80443f417ead980036aeb5b45355b1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, ignition-server, redhat-marketplace-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"84a396fab7c2ab3ae1588023933d5e474ab7b143128d353980cd15000536f48b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"8dad9a32124d40126133bba2520cede07e7f2a6cd0df33b90c1e690aeb9f3131","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"4ebee03d5cb8560c61e868d8bbb831f73868f90ba39c7b26a29c917e8892dc28","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7ce22653c1790596420acf5ec30dfcb3f90e95095df0cfef47397e47b57f8997","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"8e749cbda0987bd36e3427fb17c74e6efb4a7ffd7f85d804ab73f254f7c2b973","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4a4f87d4133fd3a9df163a91b47668459c543343ba1d123ba27a73ceaa3385ce","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"2e88e71b6a16994e8d9342a6dfffd6392efeb37bdd6569b14205b6f91d0d4a24","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b748a3b1c1b9c7fd8f7f617ddf190f4c65999a469c2f67349a1718ac3211e788","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"9597e11d65e1b6b9e701068b23031343892ee3c2b207fcc034b59895dd98d0c1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bd4fdaffaf3b15f286ad5f2ca30cb95f7541a63bca5896bf32cc0ecd46a82996","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"\u003curl\u003e","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"28d3b5e4d6e1e3b6c0885d78d546ff6d719fb6e7e972de708efbba62c246ae05","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-14..2026-08-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-25T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}